Singlefile

.github/workflows/ci_cd.yml

@@ -6,43 +6,68 @@ on:
       - main
 
 jobs:
-  confirm-changes:
-    name: confirm there are changes in the backend
+  deploy-backend:
+    name: deploy backend to tiptops server
     runs-on: ubuntu-latest
-    outputs:
-      folder_changed: ${{ steps.check_files.outputs.folder_changed }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Execute remote ssh commands using passwords
+        uses: appleboy/ssh-action@v1
         with:
-          fetch-depth: 2
-
-      - name: Find name of changes files with git
-        id: check_files
-        run: |
-          set +e
-          CHANGED_FILES=$(git diff --name-only ${{github.event.before}} ${{github.sha}} || true)
-          echo "changed files found: $CHANGED_FILES"
-
-          CHANGED_COUNT=$(echo "$CHANGED_FILES" | grep -c "backend")
+          host: ${{secrets.HOST}}
+          username: ${{secrets.USERNAME}}
+          password: ${{secrets.PASSWORD}}
+          port: ${{secrets.PORT}}
 
-          set -e
-          echo "file count that was changed: $CHANGED_COUNT"
+          script: |
+            echo "Starting deployment process"
+            # NOTE: Use 'sh' to avoid issues with inline script quoting
+            sudo -S bash -c '
+              cd /root/docker/csphere
+              #run the deployment script 
+              ./build-csphere-backend.sh
+              echo "deployment was a success"
+            ' <<< "${{secrets.PASSWORD}}"
 
-          if [ "$CHANGED_COUNT" -gt 0 ]; then
-            echo "folder_changed=true" >> $GITHUB_OUTPUT
+      - name: Slack Notification
+        if: always()
+        run: |
+          if [[ "${{ job.status }}" == "success" ]]; then
+            STATUS_TEXT="Deployment Successful"
+            COLOR="#098824"
           else
-            echo "folder_changed=false" >> $GITHUB_OUTPUT
+            STATUS_TEXT="Deployment Failed"
+            COLOR="#a80a0a"
           fi
 
-  deploy-backend:
-    name: deploy backend to tiptops server
-    runs-on: ubuntu-latest
-    # needs: confirm-changes
+          REPO_URL="https://github.com/${{ github.repository }}"
+          RUN_URL="$REPO_URL/actions/runs/${{ github.run_id }}"
+          COMMIT_URL="$REPO_URL/commit/${{ github.sha }}"
+
+          curl -X POST -H 'Content-type: application/json; charset=utf-8' \
+          -H "Authorization: Bearer ${{ secrets.SLACK_TOKEN }}" \
+          --data "{
+            \"channel\": \"${{ secrets.SLACK_CHANNEL }}\",
+            \"attachments\": [
+              {
+                \"color\": \"$COLOR\",
+                \"title\": \"Csphere CI/CD Update - Backend\",
+                \"title_link\": \"$RUN_URL\",
+                \"text\": \"$STATUS_TEXT\n\n*Repository:* ${{ github.repository }}\n*Branch:* \`${{ github.ref_name }}\` \n*Commit:* <$COMMIT_URL|${{ github.event.head_commit.message }}>\n*Author:* ${{ github.actor }}\",
+                \"footer\": \"GitHub Actions • Build #${{ github.run_number }}\",
+                \"ts\": $(date +%s)
+              }
+            ]
+          }" \
+          https://slack.com/api/chat.postMessage
+
+  deploy-worker:
+    name: deploy worker to the backend servers
+    runs-on: ununtu-latest
 
     steps:
-      - name: Execute remote ssh commands using passwords
-        # if: needs.confirm-changes.outputs.folder_changed == 'true'
+      - name: Execute remote ssh commands
         uses: appleboy/ssh-action@v1
+
         with:
           host: ${{secrets.HOST}}
           username: ${{secrets.USERNAME}}
@@ -55,6 +80,38 @@ jobs:
             sudo -S bash -c '
               cd /root/docker/csphere
               #run the deployment script 
-              ./build-csphere-backend.sh
+              ./build-csphere-worker.sh
               echo "deployment was a success"
             ' <<< "${{secrets.PASSWORD}}"
+
+      - name: Slack Notification
+        if: always()
+        run: |
+          if [[ "${{ job.status }}" == "success" ]]; then
+            STATUS_TEXT="Deployment Successful"
+            COLOR="#098824"
+          else
+            STATUS_TEXT="Deployment Failed"
+            COLOR="#a80a0a"
+          fi
+
+          REPO_URL="https://github.com/${{ github.repository }}"
+          RUN_URL="$REPO_URL/actions/runs/${{ github.run_id }}"
+          COMMIT_URL="$REPO_URL/commit/${{ github.sha }}"
+
+          curl -X POST -H 'Content-type: application/json; charset=utf-8' \
+          -H "Authorization: Bearer ${{ secrets.SLACK_TOKEN }}" \
+          --data "{
+            \"channel\": \"${{ secrets.SLACK_CHANNEL }}\",
+            \"attachments\": [
+              {
+                \"color\": \"$COLOR\",
+                \"title\": \"Csphere CI/CD Update - worker\",
+                \"title_link\": \"$RUN_URL\",
+                \"text\": \"$STATUS_TEXT\n\n*Repository:* ${{ github.repository }}\n*Branch:* \`${{ github.ref_name }}\` \n*Commit:* <$COMMIT_URL|${{ github.event.head_commit.message }}>\n*Author:* ${{ github.actor }}\",
+                \"footer\": \"GitHub Actions • Build #${{ github.run_number }}\",
+                \"ts\": $(date +%s)
+              }
+            ]
+          }" \
+          https://slack.com/api/chat.postMessage

backend/.env.example

@@ -0,0 +1,22 @@
+DATABASE_URL=
+
+SECRET_KEY=
+
+OPENROUTER_API_KEY=
+
+NEXT_PUBLIC_API_BASE_URL=http://127.0.0.1:8000
+GOOGLE_API_KEY=
+AWS_ACCESS_KEY=
+AWS_SECRET_KEY=
+BUCKET_NAME =
+GOOGLE_REDIRECT_URI=
+GOOGLE_CLIENT_ID=
+GOOGLE_CLIENT_SECRET=
+
+ACTIVEMQ_QUEUE=
+
+
+ACTIVEMQ_URL=
+ACTIVEMQ_QUEUE=
+ACTIVEMQ_USER=
+ACTIVEMQ_PASS=

backend/.gitignore

@@ -20,3 +20,6 @@ __pycache__/
 app/data_models/__pycache__/** 
 
 /dockerfile
+
+
+/archives/*

backend/alembic/env.py

@@ -38,6 +38,7 @@
 from app.data_models.folder_item import folder_item
 from app.data_models.category import Category
 from app.data_models.content_category import ContentCategory
+from app.data_models.tag import Tag
 
 target_metadata = Base.metadata
 

backend/alembic/versions/10a2bc716159_adding_html_url_for_static_web_rendering.py

@@ -0,0 +1,32 @@
+"""adding html url for static web rendering
+
+Revision ID: 10a2bc716159
+Revises: 82732e62263b
+Create Date: 2026-02-10 12:10:19.134266
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '10a2bc716159'
+down_revision: Union[str, None] = '82732e62263b'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('content', sa.Column('html_content_url', sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('content', 'html_content_url')
+    # ### end Alembic commands ###

backend/alembic/versions/1e2cab304cac_folder_embedding_column_in_folder_table.py

@@ -0,0 +1,36 @@
+"""folder_embedding column in folder table
+
+Revision ID: 1e2cab304cac
+Revises: c32fb8abe107
+Create Date: 2025-12-29 15:15:04.787853
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from pgvector.sqlalchemy import Vector
+
+
+
+# revision identifiers, used by Alembic.
+revision: str = '1e2cab304cac'
+down_revision: Union[str, None] = 'c32fb8abe107'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    #sa.Column('embedding', Vector(dim=1536), nullable=True),
+    op.add_column('folder', sa.Column('folder_embedding', Vector(dim=1536), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('folder', 'folder_embedding')
+    # ### end Alembic commands ###

backend/alembic/versions/2994a72baf17_adding_bucketing_mode_boolean_column_to_.py

@@ -0,0 +1,53 @@
+"""adding bucketing mode boolean column to Folder table
+
+Revision ID: 2994a72baf17
+Revises: 9076b42a5b56
+Create Date: 2025-12-23 12:56:56.367544
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = '2994a72baf17'
+down_revision: Union[str, None] = '9076b42a5b56'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('folder', sa.Column('bucketing_mode', sa.Boolean(), nullable=True))
+
+    op.execute("UPDATE folder SET bucketing_mode = false")
+
+    op.alter_column("folder", "bucketing_mode", nullable=False)
+
+    op.execute("UPDATE folder SET keywords = ARRAY[]::VARCHAR[]")
+
+    op.execute("UPDATE folder SET url_patterns = ARRAY[]::VARCHAR[]")
+
+    op.alter_column('folder', 'keywords',
+               existing_type=postgresql.ARRAY(sa.VARCHAR()),
+               nullable=False)
+    op.alter_column('folder', 'url_patterns',
+               existing_type=postgresql.ARRAY(sa.VARCHAR()),
+               nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('folder', 'url_patterns',
+               existing_type=postgresql.ARRAY(sa.VARCHAR()),
+               nullable=True)
+    op.alter_column('folder', 'keywords',
+               existing_type=postgresql.ARRAY(sa.VARCHAR()),
+               nullable=True)
+    op.drop_column('folder', 'bucketing_mode')
+    # ### end Alembic commands ###

backend/alembic/versions/4a23460e20b9_adding_in_foreign_contraint.py

@@ -0,0 +1,36 @@
+"""adding in foreign contraint
+
+Revision ID: 4a23460e20b9
+Revises: 55170ec071fa
+Create Date: 2026-01-23 12:11:14.992338
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '4a23460e20b9'
+down_revision: Union[str, None] = '55170ec071fa'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('content_tag', sa.Column('user_id', sa.UUID(), nullable=False))
+    op.drop_constraint(op.f('content_tag_content_id_fkey'), 'content_tag', type_='foreignkey')
+    op.create_foreign_key(None, 'content_tag', 'content_item', ['content_id', 'user_id'], ['content_id', 'user_id'])
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'content_tag', type_='foreignkey')
+    op.create_foreign_key(op.f('content_tag_content_id_fkey'), 'content_tag', 'content', ['content_id'], ['content_id'])
+    op.drop_column('content_tag', 'user_id')
+    # ### end Alembic commands ###