Dev minor

.github/workflows/build-docker.yml

@@ -0,0 +1,120 @@
+name: Build and Publish Docker Image
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to simulate (e.g., dev, dev-minor, main)'
+        required: false
+        default: 'main'
+  push:
+    branches:
+      - dev
+      - dev-minor
+
+env:
+  REGISTRY_BASE: ragtoriches
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      release_version: ${{ steps.version.outputs.RELEASE_VERSION }}
+      registry_image: ${{ steps.version.outputs.REGISTRY_IMAGE }}
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.branch || github.ref }}  # This checks out the correct branch
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install toml package
+        run: pip install toml
+
+      - name: Determine version and registry
+        id: version
+        run: |
+          VERSION=$(python -c "import toml; print(toml.load('py/pyproject.toml')['tool']['poetry']['version'])")
+          RELEASE_VERSION=$VERSION
+
+          # Use input branch if this is a workflow dispatch
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            BRANCH="${{ github.event.inputs.branch }}"
+          else
+            BRANCH="${{ github.ref }}"
+          fi
+
+          # Determine the registry based on the branch
+          if [ "$BRANCH" == "refs/heads/dev" ] || [ "$BRANCH" == "dev" ]; then
+            REGISTRY_IMAGE="${{ env.REGISTRY_BASE }}/dev"
+          elif [ "$BRANCH" == "refs/heads/dev-minor" ] || [ "$BRANCH" == "dev-minor" ]; then
+            REGISTRY_IMAGE="${{ env.REGISTRY_BASE }}/dev-minor"
+          else
+            REGISTRY_IMAGE="${{ env.REGISTRY_BASE }}/prod"
+          fi
+
+          echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_OUTPUT
+          echo "REGISTRY_IMAGE=$REGISTRY_IMAGE" >> $GITHUB_OUTPUT
+
+      - name: Set matrix
+        id: set-matrix
+        run: |
+          echo "matrix={\"include\":[{\"platform\":\"amd64\",\"runner\":\"amd2\"},{\"platform\":\"arm64\",\"runner\":\"arm2\"}]}" >> $GITHUB_OUTPUT
+
+  build:
+    needs: prepare
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJson(needs.prepare.outputs.matrix)}}
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker Auth
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.RAGTORICHES_DOCKER_UNAME }}
+          password: ${{ secrets.RAGTORICHES_DOCKER_TOKEN }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./py
+          file: ./py/Dockerfile
+          platforms: ${{ matrix.platform }}
+          push: true
+          tags: |
+            ${{ needs.prepare.outputs.REGISTRY_IMAGE }}:${{ needs.prepare.outputs.release_version }}-${{ matrix.platform }}
+            ${{ needs.prepare.outputs.REGISTRY_IMAGE }}:latest-${{ matrix.platform }}
+          provenance: false
+          sbom: false
+
+  create-manifest:
+    needs: [prepare, build]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Docker Auth
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.RAGTORICHES_DOCKER_UNAME }}
+          password: ${{ secrets.RAGTORICHES_DOCKER_TOKEN }}
+
+      - name: Create and push multi-arch manifest
+        run: |
+          docker buildx imagetools create -t ${{ needs.prepare.outputs.REGISTRY_IMAGE }}:${{ needs.prepare.outputs.release_version }} \
+            ${{ needs.prepare.outputs.REGISTRY_IMAGE }}:${{ needs.prepare.outputs.release_version }}-amd64 \
+            ${{ needs.prepare.outputs.REGISTRY_IMAGE }}:${{ needs.prepare.outputs.release_version }}-arm64
+
+          docker buildx imagetools create -t ${{ needs.prepare.outputs.REGISTRY_IMAGE }}:latest \
+            ${{ needs.prepare.outputs.REGISTRY_IMAGE }}:${{ needs.prepare.outputs.release_version }}-amd64 \
+            ${{ needs.prepare.outputs.REGISTRY_IMAGE }}:${{ needs.prepare.outputs.release_version }}-arm64
+
+      - name: Verify manifests
+        run: |
+          docker buildx imagetools inspect ${{ needs.prepare.outputs.REGISTRY_IMAGE }}:${{ needs.prepare.outputs.release_version }}
+          docker buildx imagetools inspect ${{ needs.prepare.outputs.REGISTRY_IMAGE }}:latest          

.github/workflows/publish-to-pypi.yml

@@ -2,28 +2,52 @@ name: Publish to PyPI
 
 on:
   push:
-    tags:
-      - "*"
+    branches:
+      - dev
+      - dev-minor
   workflow_dispatch:
 
 jobs:
   publish:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.10'
 
       - name: Install poetry
         working-directory: ./py
         run: pip install poetry
 
-      - name: Build and publish
+      - name: Bump version for dev branches (TestPyPI)
+        if: github.event_name == 'push'
+        working-directory: ./py
+        run: |
+          version=$(poetry version -s)
+          new_version="${version}a$(date +'%Y%m%d%H%M')"
+          poetry version $new_version
+
+      - name: Build and publish to TestPyPI
+        if: github.event_name == 'push'
+        working-directory: ./py
+        run: |
+          poetry config repositories.testpypi https://test.pypi.org/legacy/
+          poetry build
+          poetry config pypi-token.testpypi ${{ secrets.TEST_PYPI_API_TOKEN }}
+          poetry publish -r testpypi -vvv
+        env:
+          PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
+
+      - name: Build and publish to PyPI
+        if: github.event_name == 'workflow_dispatch'
         working-directory: ./py
         run: |
           poetry build
-          poetry publish --username __token__ --password ${{ secrets.PYPI_API_TOKEN }}
+          poetry config pypi-token.pypi ${{ secrets.PYPI_API_TOKEN }}
+          poetry publish -vvv
+        env:
+          PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring

docs/cookbooks/walkthrough.mdx

@@ -118,7 +118,7 @@ Key features include:
 <AccordionGroup>
 
 <Accordion icon="database" title="Ingest Data" defaultOpen={true}>
-R2R offers a powerful data ingestion process that handles various file types including `html`, `pdf`, `png`, `mp3`, and `txt`. The ingestion process parses, chunks, embeds, and stores documents efficiently with a fully asynchronous pipeline. To demonstrate this functionality:
+R2R offers a powerful data ingestion process that handles various file types including `html`, `pdf`, `png`, `mp3`, and `txt`. The full list of supported filetypes is available [here](/documentation/configuration/parsing_and_chunking). The ingestion process parses, chunks, embeds, and stores documents efficiently with a fully asynchronous pipeline. To demonstrate this functionality:
 
 <Tabs>
 <Tab title="CLI">

docs/documentation/configuration/ingestion/overview.mdx

@@ -6,7 +6,7 @@ description: 'Configure your R2R ingestion pipeline'
 
 R2R's ingestion pipeline efficiently processes various document formats, transforming them into searchable content. It seamlessly integrates with vector databases and knowledge graphs for optimal retrieval and analysis.
 
-By default, R2R leverages Unstructured's open-source [ingestion platform](https://docs.unstructured.io/open-source/introduction/overview) to handle supported file types. For formats not covered by Unstructured, such as `.mp3`, R2R implements custom ingestion logic to ensure comprehensive support.
+By default, R2R leverages Unstructured's open-source [ingestion platform](https://docs.unstructured.io/open-source/introduction/overview) to handle supported file types. For formats not covered by Unstructured, such as `.mp3`, R2R implements custom ingestion logic to ensure comprehensive support. Supported file types are listed [here](/documentation/configuration/parsing_and_chunking).
 
 ## Key Configuration Areas
 

docs/documentation/configuration/ingestion/parsing_and_chunking.mdx

@@ -21,20 +21,39 @@ Available providers:
 
 **R2R supports parsing for the following file types:**
 
+- BMP (Bitmap Image)
 - CSV (Comma-Separated Values)
+- DOC (Microsoft Word Document)
 - DOCX (Microsoft Word Document)
+- EML (Electronic Mail)
+- EPUB (Electronic Publication)
+- GIF (Graphics Interchange Format)
+- HEIC (High-Efficiency Image Format)
+- HTM (HyperText Markup)
 - HTML (HyperText Markup Language)
+- JPEG (Joint Photographic Experts Group)
+- JPG (Joint Photographic Experts Group)
 - JSON (JavaScript Object Notation)
 - MD (Markdown)
+- MSG (Microsoft Outlook Message)
+- MP3 (MPEG Audio Layer III)
+- MP4 (MPEG-4 Part 14)
+- ODT (Open Document Text)
+- ORG (Org Mode)
 - PDF (Portable Document Format)
+- P7S (PKCS#7)
+- PNG (Portable Network Graphics)
+- PPT (PowerPoint)
 - PPTX (Microsoft PowerPoint Presentation)
+- RST (reStructured Text)
+- RTF (Rich Text Format)
+- SVG (Scalable Vector Graphics)
+- TSV (Tab-Separated Values)
 - TXT (Plain Text)
+- XLS (Microsoft Excel Spreadsheet)
 - XLSX (Microsoft Excel Spreadsheet)
-- GIF (Graphics Interchange Format)
-- JPEG/JPG (Joint Photographic Experts Group)
-- PNG (Portable Network Graphics)
-- SVG (Scalable Vector Graphics)
-- MP3 (MPEG Audio Layer III)
+- XML (Extensible Markup Language)
+- TIFF (Tagged Image File Format)
 - MP4 (MPEG-4 Part 14)
 
 <Note> Parsing providers for an R2R system cannot be configured at runtime and are instead configured server side. </Note>