Merge remote-tracking branch 'origin/main' into ingestion-refactor

.github/workflows/build-main.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   build-and-publish:
-    runs-on: ubuntu-latest
+    runs-on: [ self-hosted, Linux ]
     permissions:
       packages: write
       contents: read

.github/workflows/integration-test-workflow-debian.yml

@@ -51,7 +51,7 @@ jobs:
     - name: Build and Push Docker Image
       uses: docker/build-push-action@v5
       with:
-        context: .
+        context: ./py
         file: ./py/Dockerfile
         push: true
         tags: ragtoriches/dev:latest
@@ -82,10 +82,10 @@ jobs:
         r2r documents-overview
 
         echo "Get Document Chunks"
-        r2r document-chunks 93123a68-d668-51de-8291-92162730dc87
+        r2r document-chunks --document-id=77f67c65-6406-5076-8176-3844f3ef3688
 
         echo "Delete Documents"
-        r2r delete --keys=document_id --values=b736292c-11e6-5453-9686-055da3edb866
+        r2r delete --filter="document_id:eq:f25fd516-5cac-5c09-b120-0fc841270c7e"
 
         echo "Vector Search"
         r2r search --query="What was Uber'\''s profit in 2020?"

.github/workflows/py-ci-cd.yml

@@ -1,9 +1,6 @@
 name: CI
 
 on:
-  push:
-    paths:
-      - 'py/**'
   pull_request:
     paths:
       - 'py/**'
@@ -68,7 +65,7 @@ jobs:
         run: |
           python3 -m venv venv
           source venv/bin/activate
-          poetry install
+          poetry install -E core
           pip install pytest
           poetry run pytest tests/ -k "not redis and not sentence_transformer"
           deactivate

go/README.md

@@ -0,0 +1 @@
+Our Go SDK is still under development. We welcome contributions and beta testers for it!

js/README.md

@@ -0,0 +1,42 @@
+# R2R JavaScript SDK Documentation
+
+For the complete look at the R2R JavaScript SDK, [visit our documentation.](https://r2r-docs.sciphi.ai/documentation/js-sdk/introduction)
+
+## Installation
+
+Before starting, make sure you have completed the [R2R installation](/documentation/installation).
+
+Install the R2R JavaScript SDK:
+
+```bash
+npm install r2r-js
+```
+
+## Getting Started
+
+1. Import the R2R client:
+
+```javascript
+const { r2rClient } = require('r2r-js');
+```
+
+2. Initialize the client:
+
+```javascript
+const client = new r2rClient('http://localhost:8000');
+```
+
+3. Check if R2R is running correctly:
+
+```javascript
+const healthResponse = await client.health();
+// {"status":"ok"}
+```
+
+4. Login (Optional):
+```javascript
+// client.register("[email protected]", "my_password"),
+// client.verify_email("[email protected]", "my_verification_code")
+client.login("[email protected]", "my_password")
+```
+When using authentication the commands below automatically restrict the scope to a user's available documents.

py/README.md

@@ -47,11 +47,12 @@ Then, after installing R2R, it is recommended to launch with Docker, if possible
 r2r serve --docker
 ```
 
-Alternatively, you may run R2R directly from the python package, but additional dependencies like Postgres+pgvector must be configured:
+Alternatively, you may run R2R directly from the python package, but additional dependencies like Postgres+pgvector must be configured and the full R2R core is required:
 
 ```bash
 # export OPENAI_API_KEY=sk-...
 # export POSTGRES...
+pip install 'r2r[core]'
 r2r --config-name=default serve
 ```
 
@@ -70,7 +71,7 @@ r2r --config-name=default serve
 
 ## Cookbooks
 
-- Advanced RAG Pipelines 
+- Advanced RAG Pipelines
   - [RAG Agent](https://r2r-docs.sciphi.ai/cookbooks/agent): R2R's powerful RAG agent
   - [Hybrid Search](https://r2r-docs.sciphi.ai/cookbooks/hybrid-search): Introduction to hybrid search
   - [Advanced RAG](https://r2r-docs.sciphi.ai/cookbooks/advanced-rag): Advanced RAG features

py/cli/commands/management.py

@@ -129,15 +129,28 @@ def documents_overview(client, document_ids, offset, limit):
 @click.pass_obj
 def document_chunks(client, document_id, offset, limit):
     """Get chunks of a specific document."""
+    if not document_id:
+        click.echo("Error: Document ID is required.")
+        return
+
     with timer():
-        chunks = client.document_chunks(document_id, offset, limit)
+        chunks_data = client.document_chunks(document_id, offset, limit)
+
+    chunks = chunks_data["results"]
+    if not chunks:
+        click.echo("No chunks found for the given document ID.")
+        return
 
     click.echo(f"\nNumber of chunks: {len(chunks)}")
+
     for index, chunk in enumerate(chunks, 1):
         click.echo(f"\nChunk {index}:")
-        click.echo(f"Fragment ID: {chunk['fragment_id']}")
-        click.echo(f"Text: {chunk['text'][:100]}...")
-        click.echo(f"Metadata: {chunk['metadata']}")
+        if isinstance(chunk, dict):
+            click.echo(f"Fragment ID: {chunk.get('fragment_id', 'N/A')}")
+            click.echo(f"Text: {chunk.get('text', '')[:100]}...")
+            click.echo(f"Metadata: {chunk.get('metadata', {})}")
+        else:
+            click.echo(f"Unexpected chunk format: {chunk}")
 
 
 @cli.command()

py/cli/commands/server.py

@@ -219,7 +219,6 @@ def serve(
     if docker:
 
         run_docker_serve(
-            client,
             host,
             port,
             exclude_neo4j,

py/cli/utils/docker_utils.py

@@ -11,7 +11,6 @@
 import requests
 from requests.exceptions import RequestException
 
-from core.main import R2RBuilder, R2RConfig
 from sdk import R2RClient
 
 
@@ -102,7 +101,6 @@ def run_local_serve(
 
 
 def run_docker_serve(
-    client: R2RClient,
     host: str,
     port: int,
     exclude_neo4j: bool,
@@ -119,35 +117,6 @@ def run_docker_serve(
     if config_path and config_name:
         raise ValueError("Cannot specify both config_path and config_name")
 
-    if config_path:
-        config = R2RConfig.from_toml(config_path)
-    else:
-        if not config_name:
-            config_name = "default"
-        config = R2RConfig.from_toml(R2RBuilder.CONFIG_OPTIONS[config_name])
-
-    if "unstructured" in config.parsing.provider and not image:
-        image = "emrgntcmplxty/r2r-unstructured"
-
-    completion_provider = config.completion.provider
-    completion_model = config.completion.generation_config.model
-    completion_model_provider = completion_model.split("/")[0]
-
-    check_llm_reqs(
-        completion_provider,
-        completion_model_provider,
-        include_ollama=exclude_ollama,
-    )
-
-    embedding_provider = config.embedding.provider
-    embedding_model = config.embedding.base_model
-    embedding_model_provider = embedding_model.split("/")[0]
-    check_llm_reqs(
-        embedding_provider,
-        embedding_model_provider,
-        include_ollama=exclude_ollama,
-    )
-
     no_conflict, message = check_subnet_conflict()
     if not no_conflict:
         click.secho(f"Warning: {message}", fg="red", bold=True)

py/compose.yaml

@@ -21,7 +21,7 @@ networks:
 
 services:
   r2r:
-    image: ${R2R_IMAGE:-emrgntcmplxty/r2r}
+    image: ${R2R_IMAGE:-ragtoriches/prod}
     build:
       context: .
       args: