Feat: Jira, Linear integration and GitHub importer (#1400)

* fix: silo service initial setup

* feat: moved controllers inside runner

* feat: created core definations for apps and context

* feat: added interfaces required for apps

* feat: moved shared definations

* feat: added logger as an injection for apps

* feat: changes to controllers

* feat: added vm based app execution inside manager

* feat: added dummy implementations for core constructs

* feat: moved controllers to routes

* feat: created worker moduler for processing message queue requests

* feat: added task manager defination

* feat: added context and manager separations

* feat: created defaults for the defination implementation

* feat: added async task management inside apps

* feat: removed unnecessary arguments in function calls

* feat: added context builder for app instances

* feat: added runtime type safety with zod

* feat: added packaged definition structure for silo apps

* feat: added package based configuration for engine and definitions

* feat: added custom eslint rule for throws decorator

* feat: added custom eslint rule for try catch

* feat: removed custom eslint config

* feat: added authentication controller inside engine

* feat: added auth in silo engine

* feat: migrated jira ui module to plane

* chore: removed jira app

* chore: moved engine components into src_engine directory

* feat: added base template inside new src directory

* feat: migrated worker interface with mq and redis store inside new src

* feat: created jira migrator and jira importer types

* chore: migrated worker's helper inside the base directory

* feat: added logic for booting up root worker

* feat: added jira and jira auth service inside jira app

* feat: added all transformers inside jira package

* feat: added authorization types for jira

* feat: added jira authentication oauth class

* feat: added jira transformer and pull mechanisms inside jira app

* feat: addded batching logic inside jira migrator

* feat: embedded silo sdk inside jira migrator

* feat: added plane migrator inside engine

* feat: added plane migrator for push inside migration controller

* feat: added controller methods for migration

* feat: added credentials and job routes in controller

* feat: added linear importer

* feat: added linear pull function as importer

* feat: added transformation for linear data

* feat: added pull mechanism for linear

* feat: attached linear data importer with migration controller

* fix: removed hardcoded jira from cycle and module migrator

* feat: fixed build errors

* chore: addeed example env

* feat: added authentication routes for jira

* feat: added linear route controller

* fix: restructuring

* fix: sdk configs setup

* fix: merge conflicts

* fix: sdk setup

* chore: added jira and linear importers and separate packages

* feat: moved transformation parts to linear package

* feat: decoupled jira logical parts with worker

* fix: linear silo app to use linear package

* fix: build errors and dependency resolution with packages in silo

* fix: module build errors in silo

* fix: linear authorization flow

* feat: added logic for segregated workers

* feat: attached task manager with the base starter

* feat: added migrations, query and schema into db directory

* feat: added linear importer and jira importer app structure

* feat: added silo core package

* chore: migrated worker and main engine controller inside apps

* fix: made linear integration working

* silo: added cors

* feat: added base64 state changes with jira

* chore: updated silo env

* fix: jira token cookie

* feat: added issue attachments in linear job

* feat: added credentials controller

* feat: added github package inside silo packages

* feat: added resource fetching in jira api service

* feat: added credentials locking

* feat: created resources endpoint for jira

* feat: added endpoints for getting jira data

* fix: credentials not working

* chore: exported jira types

* chore: added jira states

* fix: jira project pagination issue

* chore: initiated silo folder in web

* feat: added github routes and services in silo app

* fix: build updates

* fix: updated plane sdk and updated jira importer

* chore: updated the importer layout

* chore: removed as any from table component

* chore: integrated importer dashboard

* fix: tsup fixes

* fix: removed unnecessary files

* fix: removing tsup for building silo packages

* fix: build related issues

* fix: build issues

* fix: eslint fixes

* fix: silo build errors

* fix: silo app build errors types

* fix: reverting the cloud branch

* fix: updated package json in silo service

* fix: branch build cloud updated

* fix: build errors in apps while using sdk due to ts-alias paths

* fix: branch build cloud workflow fixes

* fix: docker compose setup updates

* fix: docker compose build fixes

* fix: docker build fixes envs updated in example file

* chore: updates folder structure and handled job services

* chore: resolved build errors in silo chore

* fix: docker compose cloud added

* fix: build process docker compose

* chore: updated jira workflow

* chore: handled the job start and jira dashboard

* chore: updated constants, file naming convention

* chore: resolved merge conflicts

* chore: integrated linear and updated jobs query

* feat: added hostname changes

* conflict: updated jira config and added issue transformation count in dashboard

* conflict: updated job

* chore: updated Jira status

* chore: updated Jira status

* fix: batch processing

* fix: batch key release

* chore: updated workflow for building images

* fix: detached silo build

* chore: updates linear and resolved build errors

* fix: batch key missing

* fix: linear workflow

* chore: updated linear queries

* fix: docker compose fixed for running silo services

---------

Co-authored-by: sriram veeraghanta <veeraghanta.sriram@gmail.com>
Co-authored-by: Henit Chobisa <chobisa.henit@gmail.com>

.env.example

@@ -44,3 +44,6 @@ NGINX_PORT=80
 SILO_BASE_URL=
 
 MONGO_DB_URL="mongodb://plane-mongodb:27017/"
+
+SILO_DB=silo
+SILO_DB_URL=postgresql://plane:plane@plane-db/silo

.github/workflows/build-branch-cloud.yml

@@ -52,6 +52,7 @@ jobs:
       build_admin: ${{ steps.changed_files.outputs.admin_any_changed }}
       build_space: ${{ steps.changed_files.outputs.space_any_changed }}
       build_live: ${{ steps.changed_files.outputs.live_any_changed }}
+      build_silo: ${{ steps.changed_files.outputs.silo_any_changed }}
       build_apiserver: ${{ steps.changed_files.outputs.apiserver_any_changed }}
       build_proxy: ${{ steps.changed_files.outputs.proxy_any_changed }}
       build_monitor: ${{ steps.changed_files.outputs.monitor_any_changed }}
@@ -60,6 +61,7 @@ jobs:
       dh_img_space: ${{ steps.set_env_variables.outputs.DH_IMG_SPACE }}
       dh_img_admin: ${{ steps.set_env_variables.outputs.DH_IMG_ADMIN }}
       dh_img_live: ${{ steps.set_env_variables.outputs.DH_IMG_LIVE }}
+      dh_img_silo: ${{ steps.set_env_variables.outputs.DH_IMG_SILO }}
       dh_img_backend: ${{ steps.set_env_variables.outputs.DH_IMG_BACKEND }}
       dh_img_proxy: ${{ steps.set_env_variables.outputs.DH_IMG_PROXY }}
       dh_img_monitor: ${{ steps.set_env_variables.outputs.DH_IMG_MONITOR }}
@@ -84,6 +86,7 @@ jobs:
           echo "DH_IMG_SPACE=space-cloud" >> $GITHUB_OUTPUT
           echo "DH_IMG_ADMIN=admin-cloud" >> $GITHUB_OUTPUT
           echo "DH_IMG_LIVE=live-cloud" >> $GITHUB_OUTPUT
+          echo "DH_IMG_SILO=silo-cloud" >> $GITHUB_OUTPUT
           echo "DH_IMG_BACKEND=backend-cloud" >> $GITHUB_OUTPUT
           echo "DH_IMG_PROXY=proxy-cloud" >> $GITHUB_OUTPUT
           echo "DH_IMG_MONITOR=monitor-cloud" >> $GITHUB_OUTPUT
@@ -165,31 +168,33 @@ jobs:
               - packages/**
               - "package.json"
               - "yarn.lock"
-              - "tsconfig.json"
               - "turbo.json"
             space:
               - space/**
               - packages/**
               - "package.json"
               - "yarn.lock"
-              - "tsconfig.json"
               - "turbo.json"
             web:
               - web/**
               - packages/**
               - "package.json"
               - "yarn.lock"
-              - "tsconfig.json"
               - "turbo.json"
             live:
               - live/**
               - packages/**
               - 'package.json'
               - 'yarn.lock'
-              - 'tsconfig.json'
               - 'turbo.json'
             monitor:
               - monitor/**
+            silo:
+              - silo/**
+              - packages/**
+              - 'package.json'
+              - 'yarn.lock'
+              - 'turbo.json'
 
   branch_build_push_admin:
     if: ${{ needs.branch_build_setup.outputs.build_admin == 'true' || github.event_name == 'workflow_dispatch' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }}
@@ -355,6 +360,33 @@ jobs:
           buildx-platforms: ${{ needs.branch_build_setup.outputs.gh_buildx_platforms }}
           buildx-endpoint: ${{ needs.branch_build_setup.outputs.gh_buildx_endpoint }}
 
+  branch_build_push_silo:
+    if: ${{ needs.branch_build_setup.outputs.build_silo == 'true' || github.event_name == 'workflow_dispatch' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }}
+    name: Build-Push Silo Docker Image
+    runs-on: ubuntu-20.04
+    needs: [branch_build_setup]
+    steps:
+      - id: checkout_files
+        name: Checkout Files
+        uses: actions/checkout@v4
+      - name: Silo Build and Push
+        uses: ./.github/actions/build-push-action
+        with:
+          build-release: ${{ needs.branch_build_setup.outputs.build_release }}
+          build-prerelease: ${{ needs.branch_build_setup.outputs.build_prerelease }}
+          release-version: ${{ needs.branch_build_setup.outputs.release_version }}
+          docker-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
+          harbor-push: ${{ needs.branch_build_setup.outputs.harbor_push }}
+          harbor-username: ${{ secrets.HARBOR_USERNAME }}
+          harbor-token: ${{ secrets.HARBOR_TOKEN }}
+          harbor-registry: ${{ vars.HARBOR_REGISTRY }}
+          harbor-project: ${{ vars.HARBOR_PROJECT }}
+          docker-image-owner: makeplane
+          docker-image-name: ${{ needs.branch_build_setup.outputs.dh_img_live }}
+          build-context: .
+          dockerfile-path: ./silo/Dockerfile.silo
+
   branch_build_push_apiserver:
     if: ${{ needs.branch_build_setup.outputs.build_apiserver == 'true' || github.event_name == 'workflow_dispatch' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }}
     name: Build-Push API Server Docker Image
@@ -396,6 +428,7 @@ jobs:
         branch_build_push_web,
         branch_build_push_space,
         branch_build_push_live,
+        branch_build_push_silo,
         branch_build_push_apiserver,
       ]
     env:

.github/workflows/build-branch-ee.yml

@@ -53,6 +53,7 @@ jobs:
       build_apiserver: ${{ steps.changed_files.outputs.apiserver_any_changed }}
       build_proxy: ${{ steps.changed_files.outputs.proxy_any_changed }}
       build_monitor: ${{ steps.changed_files.outputs.monitor_any_changed }}
+      build_silo: ${{ steps.changed_files.outputs.silo_any_changed }}
       artifact_upload_to_s3: ${{ steps.set_env_variables.outputs.artifact_upload_to_s3 }}
       artifact_s3_suffix: ${{ steps.set_env_variables.outputs.artifact_s3_suffix }}
 
@@ -63,6 +64,7 @@ jobs:
       dh_img_backend: ${{ steps.set_env_variables.outputs.DH_IMG_BACKEND }}
       dh_img_proxy: ${{ steps.set_env_variables.outputs.DH_IMG_PROXY }}
       dh_img_monitor: ${{ steps.set_env_variables.outputs.DH_IMG_MONITOR }}
+      dh_img_silo: ${{ steps.set_env_variables.outputs.DH_IMG_SILO }}
       harbor_push: ${{ steps.set_env_variables.outputs.HARBOR_PUSH }}
       build_release: ${{ steps.set_env_variables.outputs.BUILD_RELEASE }}
       build_prerelease: ${{ steps.set_env_variables.outputs.BUILD_PRERELEASE }}
@@ -93,6 +95,7 @@ jobs:
           echo "DH_IMG_BACKEND=backend-enterprise" >> $GITHUB_OUTPUT
           echo "DH_IMG_PROXY=proxy-enterprise" >> $GITHUB_OUTPUT
           echo "DH_IMG_MONITOR=monitor-enterprise" >> $GITHUB_OUTPUT
+          echo "DH_IMG_SILO=silo-enterprise" >> $GITHUB_OUTPUT
 
           BUILD_RELEASE=false
           BUILD_PRERELEASE=false
@@ -169,6 +172,13 @@ jobs:
               - "yarn.lock"
               - "tsconfig.json"
               - "turbo.json"
+            silo:
+              - silo/**
+              - packages/**
+              - 'package.json'
+              - 'yarn.lock'
+              - 'tsconfig.json'
+              - 'turbo.json'
             live:
               - live/**
               - packages/**
@@ -300,6 +310,37 @@ jobs:
           buildx-platforms: ${{ needs.branch_build_setup.outputs.gh_buildx_platforms }}
           buildx-endpoint: ${{ needs.branch_build_setup.outputs.gh_buildx_endpoint }}
 
+  branch_build_push_silo:
+    if: ${{ needs.branch_build_setup.outputs.build_silo == 'true' || github.event_name == 'workflow_dispatch' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }}
+    name: Build-Push Silo Docker Image
+    runs-on: ubuntu-20.04
+    needs: [branch_build_setup]
+    steps:
+      - id: checkout_files
+        name: Checkout Files
+        uses: actions/checkout@v4
+      - name: Silo Build and Push
+        uses: ./.github/actions/build-push-action
+        with:
+          build-release: ${{ needs.branch_build_setup.outputs.build_release }}
+          build-prerelease: ${{ needs.branch_build_setup.outputs.build_prerelease }}
+          release-version: ${{ needs.branch_build_setup.outputs.release_version }}
+          docker-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
+          harbor-push: ${{ needs.branch_build_setup.outputs.harbor_push }}
+          harbor-username: ${{ secrets.HARBOR_USERNAME }}
+          harbor-token: ${{ secrets.HARBOR_TOKEN }}
+          harbor-registry: ${{ vars.HARBOR_REGISTRY }}
+          harbor-project: ${{ vars.HARBOR_PROJECT }}
+          docker-image-owner: makeplane
+          docker-image-name: ${{ needs.branch_build_setup.outputs.dh_img_silo }}
+          build-context: .
+          dockerfile-path: ./silo/Dockerfile.silo
+          buildx-driver: ${{ needs.branch_build_setup.outputs.gh_buildx_driver }}
+          buildx-version: ${{ needs.branch_build_setup.outputs.gh_buildx_version }}
+          buildx-platforms: ${{ needs.branch_build_setup.outputs.gh_buildx_platforms }}
+          buildx-endpoint: ${{ needs.branch_build_setup.outputs.gh_buildx_endpoint }}
+
   branch_build_push_apiserver:
     if: ${{ needs.branch_build_setup.outputs.build_apiserver == 'true' || github.event_name == 'workflow_dispatch' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }}
     name: Build-Push API Server Docker Image
@@ -455,6 +496,7 @@ jobs:
         branch_build_push_apiserver,
         branch_build_push_proxy,
         branch_build_push_monitor,
+        branch_build_push_silo,
         upload_artifacts_s3,
       ]
     env:

docker-compose-cloud.yml

@@ -0,0 +1,287 @@
+x-silo-env: &silo-env
+  environment:
+    - PORT=3000
+    - BATCH_SIZE=${BATCH_SIZE:-60}
+    - MQ_PREFETCH_COUNT=${MQ_PREFETCH_COUNT:-5}
+    - APP_BASE_URL=${APP_BASE_URL:-http://web:3000}
+    - SILO_API_BASE_URL=${SILO_API_BASE_URL:-http://localhost:5050}
+    - DB_URL=${DB_URL:-postgresql://plane:plane@plane-db/silo}
+    - AMQP_URL=${AMQP_URL:-amqp://guest:guest@plane-mq:5672/}
+    - REDIS_URL=${REDIS_URL:-redis://plane-redis:6379/}
+    - SENTRY_DSN=${SENTRY_DSN:-" "}
+    - JIRA_CLIENT_ID=${JIRA_CLIENT_ID:-""}
+    - JIRA_CLIENT_SECRET=${JIRA_CLIENT_SECRET:-""}
+    - LINEAR_CLIENT_ID=${LINEAR_CLIENT_ID:-""}
+    - LINEAR_CLIENT_SECRET=${LINEAR_CLIENT_SECRET:-""}
+
+x-monitor-env: &monitor-env
+  environment:
+    - SERVICE_HTTP_WEB=web:3000
+    - SERVICE_HTTP_API=api:8000
+    - SERVICE_HTTP_LIVE=live:3000
+    - SERVICE_HTTP_PROXY=proxy:80
+    - SERVICE_HTTP_MINIO=plane-minio:9090
+    - SERVICE_TCP_REDIS=plane-redis:6379
+    - SERVICE_TCP_POSTGRES=plane-db:5432
+    - TRUSTED_PROXIES=${TRUSTED_PROXIES:-0.0.0.0/0}
+
+x-proxy-env: &proxy-env
+  environment:
+    - SITE_ADDRESS=${SITE_ADDRESS:-localhost:80}
+    - CERT_EMAIL=${CERT_EMAIL:-admin@example.com}
+    - CERT_ACME_CA=${CERT_ACME_CA:-}
+    - CERT_ACME_DNS=${CERT_ACME_DNS:-}
+    - BUCKET_NAME=${BUCKET_NAME:-uploads}
+    - FILE_SIZE_LIMIT=${FILE_SIZE_LIMIT:-5242880}
+    - LISTEN_HTTP_PORT=${LISTEN_HTTP_PORT:-80}
+    - LISTEN_HTTPS_PORT=${LISTEN_HTTPS_PORT:-443}
+
+x-app-env: &app-env
+  environment:
+    - WEB_URL=${WEB_URL:-http://localhost}
+    - DEBUG=${DEBUG:-0}
+    - SENTRY_DSN=${SENTRY_DSN:-""}
+    - SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT:-"production"}
+    - CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS:-}
+    - API_BASE_URL=${API_BASE_URL:-http://api:8000}
+    # Gunicorn Workers
+    - GUNICORN_WORKERS=${GUNICORN_WORKERS:-2}
+    #DB SETTINGS
+    - PGHOST=${PGHOST:-plane-db}
+    - PGDATABASE=${PGDATABASE:-plane}
+    - POSTGRES_USER=${POSTGRES_USER:-plane}
+    - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-plane}
+    - POSTGRES_DB=${POSTGRES_DB:-plane}
+    - PGDATA=${PGDATA:-/var/lib/postgresql/data}
+    - SILO_DB=${SILO_DB:-silo}
+    - DATABASE_URL=${DATABASE_URL:-postgresql://plane:plane@plane-db/plane}
+    # REDIS SETTINGS
+    - REDIS_HOST=${REDIS_HOST:-plane-redis}
+    - REDIS_PORT=${REDIS_PORT:-6379}
+    - REDIS_URL=${REDIS_URL:-redis://plane-redis:6379/}
+    # RabbitMQ Settings
+    - RABBITMQ_HOST=${RABBITMQ_HOST:-plane-mq}
+    - RABBITMQ_PORT=${RABBITMQ_PORT:-5672}
+    - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER:-plane}
+    - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD:-plane}
+    - RABBITMQ_DEFAULT_VHOST=${RABBITMQ_VHOST:-plane}
+    - RABBITMQ_VHOST=${RABBITMQ_VHOST:-plane}
+    - AMQP_URL=${AMQP_URL:-amqp://plane:plane@plane-mq:5672/plane}
+    # Application secret
+    - SECRET_KEY=${SECRET_KEY:-60gp0byfz2dvffa45cxl20p1scy9xbpf6d8c5y0geejgkyp1b5}
+    # DATA STORE SETTINGS
+    - USE_MINIO=${USE_MINIO:-1}
+    - AWS_REGION=${AWS_REGION:-""}
+    - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-"access-key"}
+    - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-"secret-key"}
+    - AWS_S3_ENDPOINT_URL=${AWS_S3_ENDPOINT_URL:-http://plane-minio:9000}
+    - AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME:-uploads}
+    - MINIO_ROOT_USER=${MINIO_ROOT_USER:-"access-key"}
+    - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-"secret-key"}
+    - BUCKET_NAME=${BUCKET_NAME:-uploads}
+    - FILE_SIZE_LIMIT=${FILE_SIZE_LIMIT:-5242880}
+    - SSL_VERIFY=${SSL_VERIFY:-1}
+    - FEATURE_FLAG_SERVER_BASE_URL=${FEATURE_FLAG_SERVER_BASE_URL:-http://monitor:8080}
+    - PAYMENT_SERVER_BASE_URL=${PAYMENT_SERVER_BASE_URL:-http://monitor:8080}
+
+x-live-env: &live-env
+  environment:
+    - API_BASE_URL=${API_BASE_URL:-http://api:8000}
+    - REDIS_HOST=${REDIS_HOST:-plane-redis}
+    - REDIS_PORT=${REDIS_PORT:-6379}
+    - REDIS_URL=${REDIS_URL:-redis://plane-redis:6379/}
+
+services:
+  admin:
+    <<: *app-env
+    image: registry.plane.tools/plane/admin-enterprise:${APP_RELEASE_VERSION}
+    restart: unless-stopped
+    command: node admin/server.js admin
+    deploy:
+      replicas: ${ADMIN_REPLICAS:-1}
+    depends_on:
+      - api
+      - web
+
+  silo:
+    <<: *silo-env
+    image: registry.plane.tools/plane/silo-enterprise:${APP_RELEASE_VERSION}
+    restart: always
+    command: node silo/start.js
+    depends_on: 
+      plane-mq:
+        condition: service_healthy
+      silo_migrator:
+        condition: service_completed_successfully
+
+  silo_migrator:
+    <<: *silo-env
+    image: registry.plane.tools/plane/silo-enterprise:${APP_RELEASE_VERSION}
+    restart: "no"
+    command: npm --prefix ./silo run db:migrate
+    depends_on: 
+      plane-db:
+        condition: service_healthy
+      plane-mq:
+        condition: service_healthy
+
+  web:
+    <<: *app-env
+    image: registry.plane.tools/plane/web-enterprise:${APP_RELEASE_VERSION}
+    restart: unless-stopped
+    command: node web/server.js web
+    deploy:
+      replicas: ${WEB_REPLICAS:-1}
+    depends_on:
+      - api
+      - worker
+
+  space:
+    <<: *app-env
+    image: registry.plane.tools/plane/space-enterprise:${APP_RELEASE_VERSION}
+    restart: unless-stopped
+    command: node space/server.js space
+    deploy:
+      replicas: ${SPACE_REPLICAS:-1}
+    depends_on:
+      - api
+      - web
+
+  live:
+    <<: *live-env
+    image: registry.plane.tools/plane/live-enterprise:${APP_RELEASE_VERSION}
+    restart: unless-stopped
+    command: node live/dist/server.js live
+    deploy:
+      replicas: ${LIVE_REPLICAS:-1}
+    depends_on:
+      - api
+      - web
+
+  monitor:
+    <<: *monitor-env
+    image: registry.plane.tools/plane/monitor-enterprise:${APP_RELEASE_VERSION}
+    restart: on-failure:5
+    volumes:
+      - ${INSTALL_DIR}/data/monitor:/app
+
+  api:
+    <<: *app-env
+    image: registry.plane.tools/plane/backend-enterprise:${APP_RELEASE_VERSION}
+    restart: unless-stopped
+    command: ./bin/docker-entrypoint-api-ee.sh
+    deploy:
+      replicas: ${API_REPLICAS:-1}
+    volumes:
+      - ${INSTALL_DIR}/logs/api:/code/plane/logs
+    depends_on:
+      - plane-db
+      - plane-redis
+
+  worker:
+    <<: *app-env
+    image: registry.plane.tools/plane/backend-enterprise:${APP_RELEASE_VERSION}
+    restart: unless-stopped
+    command: ./bin/docker-entrypoint-worker.sh
+    volumes:
+      - ${INSTALL_DIR}/logs/worker:/code/plane/logs
+    depends_on:
+      - api
+      - plane-db
+      - plane-redis
+
+  beat-worker:
+    <<: *app-env
+    image: registry.plane.tools/plane/backend-enterprise:${APP_RELEASE_VERSION}
+    restart: unless-stopped
+    command: ./bin/docker-entrypoint-beat.sh
+    volumes:
+      - ${INSTALL_DIR}/logs/beat-worker:/code/plane/logs
+    depends_on:
+      - api
+      - plane-db
+      - plane-redis
+
+  migrator:
+    <<: *app-env
+    image: registry.plane.tools/plane/backend-enterprise:${APP_RELEASE_VERSION}
+    restart: "no"
+    command: ./bin/docker-entrypoint-migrator.sh
+    volumes:
+      - ${INSTALL_DIR}/logs/migrator:/code/plane/logs
+    depends_on:
+      - plane-db
+      - plane-redis
+
+  plane-db:
+    <<: *app-env
+    image: registry.plane.tools/plane/postgres:15.5-alpine
+    restart: unless-stopped
+    command: >
+      bash -c '
+        docker-entrypoint.sh postgres &
+        sleep 5 &&
+        result=$(PGPASSWORD=${POSTGRES_PASSWORD} psql -U $POSTGRES_USER -tAc "SELECT 1 FROM pg_database WHERE datname='${SILO_DB}'") &&
+        echo "$result" | grep -q 1 && result=0 || result=1 &&
+        if [ $result -eq 0 ]; then
+          echo "Database '${SILO_DB}' already exists"
+        else
+          PGPASSWORD=${POSTGRES_PASSWORD} psql -U $POSTGRES_USER -c "CREATE DATABASE ${SILO_DB};"
+          echo "Created database '${SILO_DB}'"
+        fi &&
+        wait
+      '
+    healthcheck:
+      test: ["CMD-SHELL", "PGPASSWORD=${POSTGRES_PASSWORD} psql -U $POSTGRES_USER -tAc \"SELECT CASE WHEN EXISTS(SELECT 1 FROM pg_database WHERE datname='${SILO_DB}') THEN 1 ELSE 0 END AS ${SILO_DB}, CASE WHEN EXISTS(SELECT 1 FROM pg_database WHERE datname='${POSTGRES_DB}') THEN 1 ELSE 0 END AS ${POSTGRES_DB}\" | grep -q '1|1' && echo 'Both databases exist' || (echo 'One or both databases are missing' && exit 1)"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    # volumes:
+    #   - ${INSTALL_DIR}/data/db:/var/lib/postgresql/data
+
+  plane-redis:
+    <<: *app-env
+    image: registry.plane.tools/plane/valkey:7.2.5-alpine
+    restart: unless-stopped
+    volumes:
+      - ${INSTALL_DIR}/data/redis:/data
+
+  plane-mq:
+    <<: *app-env
+    image: registry.plane.tools/plane/rabbitmq:3.13.6-management-alpine
+    restart: unless-stopped
+    healthcheck:
+      test: rabbitmq-diagnostics -q ping
+      interval: 10s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - ${INSTALL_DIR}/data/mq:/var/lib/rabbitmq
+
+  plane-minio:
+    <<: *app-env
+    image: registry.plane.tools/plane/minio:latest
+    restart: unless-stopped
+    command: server /export --console-address ":9090"
+    volumes:
+      - ${INSTALL_DIR}/data/minio/uploads:/export
+      - ${INSTALL_DIR}/data/minio/data:/data
+
+  # Comment this if you already have a reverse proxy running
+  proxy:
+    <<: *proxy-env
+    image: registry.plane.tools/plane/caddy:latest
+    restart: unless-stopped
+    ports:
+      - ${LISTEN_HTTP_PORT:-80}:${LISTEN_HTTP_PORT:-80}
+      - ${LISTEN_HTTPS_PORT:-443}:${LISTEN_HTTPS_PORT:-443}
+    volumes:
+      - ${INSTALL_DIR}/Caddyfile:/etc/caddy/Caddyfile
+      - ${INSTALL_DIR}/caddy/config:/config
+      - ${INSTALL_DIR}/caddy/data:/data
+    depends_on:
+      - web
+      - api
+      - space
+      - admin
+      - live

docker-compose.yml

@@ -1,3 +1,19 @@
+x-silo-env: &silo-env
+  environment:
+    - PORT=3000
+    - BATCH_SIZE=${BATCH_SIZE:-60}
+    - MQ_PREFETCH_COUNT=${MQ_PREFETCH_COUNT:-5}
+    - APP_BASE_URL=${APP_BASE_URL:-http://web:3000}
+    - SILO_API_BASE_URL=${SILO_API_BASE_URL:-http://localhost:5050}
+    - DB_URL=${DB_URL:-postgresql://plane:plane@plane-db/silo}
+    - AMQP_URL=${AMQP_URL:-amqp://guest:guest@plane-mq:5672/}
+    - REDIS_URL=${REDIS_URL:-redis://plane-redis:6379/}
+    - SENTRY_DSN=${SENTRY_DSN:-" "}
+    - JIRA_CLIENT_ID=${JIRA_CLIENT_ID:-""}
+    - JIRA_CLIENT_SECRET=${JIRA_CLIENT_SECRET:-""}
+    - LINEAR_CLIENT_ID=${LINEAR_CLIENT_ID:-""}
+    - LINEAR_CLIENT_SECRET=${LINEAR_CLIENT_SECRET:-""}
+
 services:
   web:
     container_name: web
@@ -37,6 +53,43 @@ services:
       - api
       - web
 
+  live:
+    container_name: live
+    build:
+      context: .
+      dockerfile: ./live/Dockerfile.live
+      args:
+        DOCKER_BUILDKIT: 1
+    restart: always
+    command: node live/dist/server.js
+
+  silo:
+    <<: *silo-env
+    container_name: silo
+    build:
+      context: .
+      dockerfile: ./silo/Dockerfile.silo
+      args:
+        DOCKER_BUILDKIT: 1
+    restart: always
+    command: >
+      /bin/sh -c "
+        npm --prefix ./silo run db:migrate
+        node silo/start.js
+      "
+    depends_on: 
+      plane-mq:
+        condition: service_healthy
+
+  # monitor:
+  #   container_name: monitor
+  #   build:
+  #     context: .
+  #     dockerfile: ./monitor/Dockerfile
+  #     args:
+  #       DOCKER_BUILDKIT: 1
+  #   restart: always
+
   api:
     container_name: api
     build:
@@ -45,7 +98,7 @@ services:
       args:
         DOCKER_BUILDKIT: 1
     restart: always
-    command: ./bin/docker-entrypoint-api.sh
+    command: ./bin/docker-entrypoint-api-ee.sh
     env_file:
       - ./apiserver/.env
     depends_on:
@@ -108,21 +161,17 @@ services:
       - plane-db
       - plane-redis
 
-  live:
-    container_name: plane-live
-    build:
-      context: .
-      dockerfile: ./live/Dockerfile.live
-      args:
-        DOCKER_BUILDKIT: 1
-    restart: always
-    command: node live/dist/server.js
-
   plane-db:
     container_name: plane-db
     image: postgres:15.7-alpine
     restart: always
-    command: postgres -c 'max_connections=1000'
+    command: >
+      bash -c "
+        docker-entrypoint.sh postgres &
+        sleep 10
+        psql -U $$POSTGRES_USER -d $$POSTGRES_DB -c 'CREATE DATABASE silo;'
+        wait
+      "
     volumes:
       - pgdata:/var/lib/postgresql/data
     env_file:
@@ -132,6 +181,12 @@ services:
       POSTGRES_DB: ${POSTGRES_DB}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
       PGDATA: /var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
 
   plane-redis:
     container_name: plane-redis
@@ -142,14 +197,21 @@ services:
 
   plane-mq:
     container_name: plane-mq
-    image: rabbitmq:3.13.6-management-alpine
+    image: rabbitmq:management
     restart: always
+    ports:
+      - "15672:15672"
     env_file:
       - .env
     environment:
       RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER}
       RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD}
       RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST}
+    healthcheck:
+      test: rabbitmq-diagnostics -q ping
+      interval: 10s
+      timeout: 10s
+      retries: 5
     volumes:
       - rabbitmq_data:/var/lib/rabbitmq
 

package.json

@@ -9,6 +9,8 @@
     "space",
     "admin",
     "live",
+    "silo",
+    "packages/silo/*",
     "packages/*"
   ],
   "scripts": {

packages/constants/index.ts

@@ -1,3 +1,4 @@
 export * from "./auth";
 export * from "./issue";
 export * from "./payment";
+export * from "./silo"

packages/constants/silo.ts

@@ -0,0 +1,20 @@
+export const JIRA_SCOPES = [
+  "offline_access",
+  "read:jira-work",
+  "read:me",
+  "read:jira-user",
+  "read:workflow:jira",
+  "read:board-scope:jira-software",
+  "read:project:jira",
+  "read:epic:jira-software",
+  "read:sprint:jira-software",
+  "read:issue-details:jira",
+  "read:jql:jira",
+  "read:project.component:jira",
+  "read:group:jira",
+  "read:application-role:jira",
+  "read:avatar:jira",
+  "read:user:jira",
+  "read:attachment:jira",
+  "read:issue-meta:jira",
+];

packages/eslint-config-custom/package.json

@@ -1,18 +0,0 @@
-{
-  "name": "eslint-config-custom",
-  "private": true,
-  "version": "1.3.1",
-  "main": "index.js",
-  "license": "MIT",
-  "dependencies": {
-    "@typescript-eslint/eslint-plugin": "^7.1.1",
-    "@typescript-eslint/parser": "^7.1.1",
-    "eslint": "^8.57.0",
-    "eslint-config-next": "^14.1.0",
-    "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.12.4",
-    "eslint-plugin-import": "^2.29.1",
-    "eslint-plugin-react": "^7.33.2",
-    "typescript": "5.4.5"
-  }
-}

packages/eslint-config/next.js

@@ -76,6 +76,11 @@ module.exports = {
             group: "external",
             position: "after",
           },
+          {
+            pattern: "@silo/**",
+            group: "external",
+            position: "after",
+          },
           {
             pattern: "@/**",
             group: "internal",

packages/helpers/package.json

@@ -15,7 +15,7 @@
   "devDependencies": {
     "@types/node": "^22.5.4",
     "@types/react": "^18.3.5",
-    "typescript": "^5.6.2",
+    "typescript": "^5.3.3",
     "tsup": "^7.2.0"
   },
   "dependencies": {

packages/sdk/.eslintrc.js

@@ -0,0 +1,36 @@
+const { resolve } = require("node:path");
+const project = resolve(process.cwd(), "tsconfig.json");
+
+module.exports = {
+  root: true,
+  extends: ["custom"],
+  parser: "@typescript-eslint/parser",
+  settings: {
+    "import/resolver": {
+      typescript: {
+        project,
+      },
+      node: {
+        moduleDirectory: ["node_modules", "."],
+      },
+    },
+  },
+  parserOptions: {
+    ecmaVersion: 2020,
+    sourceType: "module",
+    project: project,
+  },
+  rules: {
+    "import/order": [
+      "error",
+      {
+        groups: ["builtin", "external", "internal", "parent", "sibling"],
+        pathGroupsExcludedImportTypes: ["builtin", "internal", "react"],
+        alphabetize: {
+          order: "asc",
+          caseInsensitive: true,
+        },
+      },
+    ],
+  },
+};

packages/sdk/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@plane/sdk",
+  "version": "0.23.0",
+  "private": true,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "dev": "tsc --watch",
+    "build": "tsc && tsc-alias",
+    "lint": "eslint --ext .ts src"
+  },
+  "dependencies": {
+    "axios": "^1.7.7",
+    "lodash": "^4.17.21"
+  },
+  "devDependencies": {
+    "@plane/eslint-config": "*",
+    "@plane/typescript-config": "*",
+    "tsc-alias": "^1.8.10"
+  }
+}

packages/sdk/src/client.ts

@@ -0,0 +1,34 @@
+import { CycleService } from "@/services/cycle.service";
+import { IssueCommentService } from "@/services/issue-comment.service";
+import { IssueService } from "@/services/issue.service";
+import { LabelService } from "@/services/label.service";
+import { ModuleService } from "@/services/module.service";
+import { ProjectService } from "@/services/project.service";
+import { StateService } from "@/services/state.service";
+import { UserService } from "@/services/user.service";
+// types
+import { ClientOptions } from "@/types/types";
+
+export class Client {
+  options: ClientOptions;
+  users: UserService;
+  label: LabelService;
+  state: StateService;
+  issue: IssueService;
+  cycles: CycleService;
+  modules: ModuleService;
+  project: ProjectService;
+  issueComment: IssueCommentService;
+
+  constructor(options: ClientOptions) {
+    this.options = options;
+    this.label = new LabelService(options);
+    this.state = new StateService(options);
+    this.issue = new IssueService(options);
+    this.users = new UserService(options);
+    this.project = new ProjectService(options);
+    this.issueComment = new IssueCommentService(options);
+    this.cycles = new CycleService(options);
+    this.modules = new ModuleService(options);
+  }
+}

packages/sdk/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./client";
+export * from "./types";

packages/sdk/src/lib/constants.ts

@@ -0,0 +1,15 @@
+export const PLANE_PRIORITIES = [
+  "urgent",
+  "high",
+  "medium",
+  "low",
+  "none",
+] as const;
+
+export const generateHexCode = () => {
+  const hexCode = Math.floor(Math.random() * 0xffffff)
+    .toString(16)
+    .padStart(6, "0");
+  return `#${hexCode}`;
+};
+

packages/sdk/src/services/api.service.ts

@@ -0,0 +1,52 @@
+import axios, { AxiosInstance } from "axios";
+// types
+import { ClientOptions } from "@/types/types";
+
+export abstract class APIService {
+  private axiosInstance: AxiosInstance;
+
+  constructor(options: ClientOptions) {
+    const { baseURL } = options;
+    this.axiosInstance = axios.create({
+      baseURL,
+      headers: { "X-API-Key": options.apiToken },
+    });
+    this.setupInterceptors();
+  }
+
+  private setupInterceptors() {
+    this.axiosInstance.interceptors.response.use(
+      (response) => response,
+      (error) => {
+        if (error.response && error.response.status === 401) {
+          console.log("401 error");
+        }
+        return Promise.reject(error);
+      }
+    );
+  }
+
+  get(url: string, config = {}) {
+    return this.axiosInstance.get(url, config);
+  }
+
+  post(url: string, data = {}, config = {}) {
+    return this.axiosInstance.post(url, data, config);
+  }
+
+  put(url: string, data = {}, config = {}) {
+    return this.axiosInstance.put(url, data, config);
+  }
+
+  patch(url: string, data = {}, config = {}) {
+    return this.axiosInstance.patch(url, data, config);
+  }
+
+  delete(url: string, data?: any, config = {}) {
+    return this.axiosInstance.delete(url, { data, ...config });
+  }
+
+  request(config = {}) {
+    return this.axiosInstance(config);
+  }
+}

packages/sdk/src/services/cycle.service.ts

@@ -0,0 +1,80 @@
+import { APIService } from "@/services/api.service";
+// types
+import {
+  ClientOptions,
+  ExcludedProps,
+  ExCycle,
+  Optional,
+  Paginated,
+} from "@/types/types";
+
+export class CycleService extends APIService {
+  constructor(options: ClientOptions) {
+    super(options);
+  }
+
+  async list(slug: string, projectId: string): Promise<Paginated<ExCycle>> {
+    return this.get(`/api/v1/workspaces/${slug}/projects/${projectId}/cycles/`)
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async create(
+    slug: string,
+    projectId: string,
+    payload: Omit<Optional<ExCycle>, ExcludedProps>
+  ): Promise<ExCycle> {
+    return this.post(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/cycles/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async update(
+    slug: string,
+    projectId: string,
+    cycleId: string,
+    payload: Omit<Optional<ExCycle>, ExcludedProps>
+  ) {
+    return this.patch(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/cycles/${cycleId}/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async destroy(slug: string, projectId: string, cycleId: string) {
+    return this.delete(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/cycles/${cycleId}/`
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async addIssues(
+    slug: string,
+    projectId: string,
+    cycleId: string,
+    issueIds: string[]
+  ) {
+    return this.post(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/cycles/${cycleId}/cycle-issues/`,
+      { issues: issueIds }
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+}

packages/sdk/src/services/issue-comment.service.ts

@@ -0,0 +1,78 @@
+import { APIService } from "@/services/api.service";
+// types
+import {
+  ClientOptions,
+  ExcludedProps,
+  ExIssueComment,
+  ExIssueLabel,
+  Optional,
+  Paginated,
+} from "@/types/types";
+
+export class IssueCommentService extends APIService {
+  constructor(options: ClientOptions) {
+    super(options);
+  }
+
+  async list(
+    slug: string,
+    projectId: string,
+    issueId: string
+  ): Promise<Paginated<ExIssueLabel>> {
+    return this.get(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/issues/${issueId}/comments/`
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async create(
+    slug: string,
+    projectId: string,
+    issueId: string,
+    payload: Omit<Optional<ExIssueComment>, ExcludedProps>
+  ) {
+    return this.post(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/issues/${issueId}/comments/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async update(
+    slug: string,
+    projectId: string,
+    issueId: string,
+    commentId: string,
+    payload: Omit<Optional<ExIssueComment>, ExcludedProps>
+  ) {
+    return this.patch(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/issues/${issueId}/comments/${commentId}/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async destroy(
+    slug: string,
+    projectId: string,
+    issueId: string,
+    commentId: string
+  ) {
+    return this.delete(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/issues/${issueId}/comments/${commentId}/`
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+}

packages/sdk/src/services/issue.service.ts

@@ -0,0 +1,130 @@
+import { APIService } from "@/services/api.service";
+// types
+import {
+  ClientOptions,
+  ExcludedProps,
+  ExIssue,
+  ExIssueAttachment,
+  Optional,
+  Paginated,
+} from "@/types/types";
+
+export class IssueService extends APIService {
+  constructor(options: ClientOptions) {
+    super(options);
+  }
+
+  async list(slug: string, projectId: string): Promise<Paginated<ExIssue>> {
+    return this.get(`/api/v1/workspaces/${slug}/projects/${projectId}/issues/`)
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async create(
+    slug: string,
+    projectId: string,
+    payload: Omit<Optional<ExIssue>, ExcludedProps>
+  ): Promise<ExIssue> {
+    return this.post(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/issues/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async update(
+    slug: string,
+    projectId: string,
+    issueId: string,
+    payload: Omit<Optional<ExIssue>, ExcludedProps>
+  ) {
+    return this.patch(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/issues/${issueId}/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async destroy(slug: string, projectId: string, issueId: string) {
+    return this.delete(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/issues/${issueId}/`
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async createLink(
+    slug: string,
+    projectId: string,
+    issueId: string,
+    title: string,
+    url: string
+  ) {
+    return this.post(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/issues/${issueId}/links/`,
+      {
+        title: title,
+        url: url,
+      }
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async uploadIssueAttachment(
+    workspaceSlug: string,
+    projectId: string,
+    issueId: string,
+    file: FormData
+  ): Promise<ExIssueAttachment> {
+    return this.post(
+      `/api/v1/workspaces/${workspaceSlug}/projects/${projectId}/issues/${issueId}/issue-attachments/`,
+      file
+    )
+      .then((response) => response?.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async getIssueWithExternalId(
+    workspaceSlug: string,
+    projectId: string,
+    externalId: string,
+    externalSource: string
+  ): Promise<ExIssue> {
+    return this.get(
+      `/api/v1/workspaces/${workspaceSlug}/projects/${projectId}/issues/?external_id=${externalId}&external_source=${externalSource}`
+    )
+      .then((response) => response?.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async getIssueAttachments(
+    workspaceSlug: string,
+    projectId: string,
+    issueId: string
+  ): Promise<ExIssueAttachment[]> {
+    return this.get(
+      `/api/v1/workspaces/${workspaceSlug}/projects/${projectId}/issues/${issueId}/issue-attachments/`
+    )
+      .then((response) => response?.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+}

packages/sdk/src/services/label.service.ts

@@ -0,0 +1,67 @@
+import { APIService } from "@/services/api.service";
+// types
+import {
+  ClientOptions,
+  ExcludedProps,
+  ExIssueLabel,
+  Optional,
+  Paginated,
+} from "@/types/types";
+
+export class LabelService extends APIService {
+  constructor(options: ClientOptions) {
+    super(options);
+  }
+
+  async list(
+    slug: string,
+    projectId: string
+  ): Promise<Paginated<ExIssueLabel>> {
+    return this.get(`/api/v1/workspaces/${slug}/projects/${projectId}/labels/`)
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error;
+      });
+  }
+
+  async create(
+    slug: string,
+    projectId: string,
+    payload: Omit<Optional<ExIssueLabel>, ExcludedProps>
+  ): Promise<ExIssueLabel> {
+    return this.post(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/labels/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error;
+      });
+  }
+
+  async update(
+    slug: string,
+    projectId: string,
+    labelId: string,
+    payload: Omit<Optional<ExIssueLabel>, ExcludedProps>
+  ) {
+    return this.patch(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/labels/${labelId}/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error;
+      });
+  }
+
+  async destroy(slug: string, projectId: string, labelId: string) {
+    return this.delete(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/labels/${labelId}/`
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error;
+      });
+  }
+}

packages/sdk/src/services/module.service.ts

@@ -0,0 +1,81 @@
+import { APIService } from "@/services/api.service";
+// types
+import {
+  ClientOptions,
+  ExcludedProps,
+  ExModule,
+  Optional,
+  Paginated,
+} from "@/types/types";
+
+export class ModuleService extends APIService {
+  constructor(options: ClientOptions) {
+    super(options);
+  }
+
+  async list(slug: string, projectId: string): Promise<Paginated<ExModule>> {
+    return this.get(`/api/v1/workspaces/${slug}/projects/${projectId}/modules/`)
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async create(
+    slug: string,
+    projectId: string,
+    payload: Omit<Optional<ExModule>, ExcludedProps>
+  ): Promise<ExModule> {
+    return this.post(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/modules/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async update(
+    slug: string,
+    projectId: string,
+    moduleId: string,
+    payload: Omit<Optional<ExModule>, ExcludedProps>
+  ) {
+    return this.patch(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/modules/${moduleId}/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async destroy(slug: string, projectId: string, moduleId: string) {
+    return this.delete(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/modules/${moduleId}/`
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async addIssues(
+    slug: string,
+    projectId: string,
+    moduleId: string,
+    moduleName: string,
+    issueIds: string[]
+  ) {
+    return this.post(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/modules/${moduleId}/module-issues/`,
+      { name: moduleName, issues: issueIds }
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+}

packages/sdk/src/services/project.service.ts

@@ -0,0 +1,34 @@
+import { APIService } from "@/services/api.service";
+// types
+import {
+  ClientOptions,
+  ExcludedProps,
+  ExProject,
+  Optional,
+  Paginated,
+} from "@/types/types";
+
+export class ProjectService extends APIService {
+  constructor(options: ClientOptions) {
+    super(options);
+  }
+
+  async list(slug: string): Promise<Paginated<ExProject>> {
+    return this.get(`/api/v1/workspaces/${slug}/projects/`)
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async create(
+    slug: string,
+    payload: Omit<Optional<ExProject>, ExcludedProps>
+  ) {
+    return this.post(`/api/v1/workspaces/${slug}/projects/`, payload)
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+}

packages/sdk/src/services/state.service.ts

@@ -0,0 +1,64 @@
+import { APIService } from "@/services/api.service";
+// types
+import {
+  ClientOptions,
+  ExcludedProps,
+  Optional,
+  Paginated,
+  ExState,
+} from "@/types/types";
+
+export class StateService extends APIService {
+  constructor(options: ClientOptions) {
+    super(options);
+  }
+
+  async list(slug: string, projectId: string): Promise<Paginated<ExState>> {
+    return this.get(`/api/v1/workspaces/${slug}/projects/${projectId}/states/`)
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async create(
+    slug: string,
+    projectId: string,
+    payload: Omit<Optional<ExState>, ExcludedProps>
+  ): Promise<ExState> {
+    return this.post(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/states/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async update(
+    slug: string,
+    projectId: string,
+    stateId: string,
+    payload: Omit<Optional<ExState>, ExcludedProps>
+  ) {
+    return this.patch(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/states/${stateId}/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async destroy(slug: string, projectId: string, stateId: string) {
+    return this.delete(
+      `/api/v1/workspaces/${slug}/projects/${projectId}/states/${stateId}/`
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+}

packages/sdk/src/services/user.service.ts

@@ -0,0 +1,39 @@
+import { APIService } from "@/services/api.service";
+// types
+import {
+  ClientOptions,
+  PlaneUser,
+  UserCreatePayload,
+  UserResponsePayload,
+} from "@/types/types";
+
+export class UserService extends APIService {
+  constructor(options: ClientOptions) {
+    super(options);
+  }
+
+  async create(
+    workspaceSlug: string,
+    projectId: string,
+    payload: UserCreatePayload
+  ): Promise<UserResponsePayload> {
+    return this.post(
+      `/api/v1/workspaces/${workspaceSlug}/projects/${projectId}/members/`,
+      payload
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  async list(workspaceSlug: string, projectId: string): Promise<PlaneUser[]> {
+    return this.get(
+      `/api/v1/workspaces/${workspaceSlug}/projects/${projectId}/members/`
+    )
+      .then((response) => response.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+}

packages/sdk/src/types/index.ts

@@ -0,0 +1 @@
+export * from "./types"

packages/sdk/src/types/types.ts

@@ -0,0 +1,277 @@
+// service types
+export type ClientOptions = {
+  baseURL: string;
+  apiToken: string;
+};
+
+/* ----------------- utility --------------------- */
+type ExBase = {
+  project: string;
+  workspace: string;
+  parent: string | null;
+
+  external_id: string;
+  external_source: string;
+
+  updated_by: string;
+  created_by: string;
+
+  created_at: string;
+  update_at: string;
+};
+
+export type Optional<T extends object> = {
+  [K in keyof T]?: T[K];
+};
+
+export type Paginated<T> = {
+  grouped_by: null | string;
+  sub_grouped_by: null | string;
+  total_count: number;
+  next_cursor: string;
+  prev_cursor: string;
+  next_page_results: boolean;
+  prev_page_results: boolean;
+  count: number;
+  total_pages: number;
+  total_results: number;
+  results: T[];
+};
+
+export type ExcludedProps =
+  | "id"
+  | "created_at"
+  | "created_by"
+  | "updated_at"
+  | "updated_by";
+
+/* ----------------- Base Types --------------------- */
+type IIssueLabel = {
+  id: string;
+  name: string;
+  color: string;
+  parent: string | null;
+  sort_order: number;
+};
+
+export type IIssueComment = {
+  id: string;
+  issue: string;
+  actor: string;
+  comment_html: string;
+  access: string;
+  is_member: boolean;
+};
+
+type IIsssue = {
+  id: string;
+  updated_at: string;
+  point: any;
+  name: string;
+  description_html: string;
+  description_binary: any;
+  priority: string;
+  start_date: string;
+  target_date: string;
+  sequence_id: number;
+  sort_order: number;
+  completed_at: any;
+  archived_at: any;
+  is_draft: boolean;
+  state: string;
+  estimate_point: any;
+  assignees: string[];
+  labels: string[];
+};
+
+export type TStateGroups =
+  | "backlog"
+  | "unstarted"
+  | "started"
+  | "completed"
+  | "cancelled";
+
+export interface IState {
+  id: string;
+  color: string;
+  default: boolean;
+  description: string;
+  group: TStateGroups;
+  name: string;
+  sequence: number;
+}
+
+export type TModuleStatus =
+  | "backlog"
+  | "planned"
+  | "in-progress"
+  | "paused"
+  | "completed"
+  | "cancelled";
+
+export interface IModule {
+  total_issues: number;
+  completed_issues: number;
+  backlog_issues: number;
+  started_issues: number;
+  unstarted_issues: number;
+  cancelled_issues: number;
+  total_estimate_points?: number;
+  completed_estimate_points?: number;
+  backlog_estimate_points: number;
+  started_estimate_points: number;
+  unstarted_estimate_points: number;
+  cancelled_estimate_points: number;
+
+  id: string;
+  name: string;
+  description: string;
+  description_text: any;
+  description_html: any;
+  lead: string | null;
+  members: string[];
+  // link_module?: ILinkDetails[];
+  sub_issues?: number;
+  is_favorite: boolean;
+  sort_order: number;
+  // view_props: {
+  // 	filters: IIssueFilterOptions;
+  // };
+  status?: TModuleStatus;
+  archived_at: string | null;
+  start_date: string | null;
+  target_date: string | null;
+}
+
+export interface ICycle {
+  id: string;
+  total_issues: number;
+  cancelled_issues: number;
+  completed_issues: number;
+  started_issues: number;
+  unstarted_issues: number;
+  backlog_issues: number;
+  created_at: string;
+  updated_at: string;
+  name: string;
+  description: string;
+  start_date: string | null;
+  end_date: string | null;
+  view_props: Record<string, any>;
+  sort_order: number;
+  progress_snapshot: Record<string, any>;
+  archived_at: string | null;
+  logo_props: Record<string, any>;
+  owned_by: string;
+}
+
+export type PlaneEntities = {
+  labels: Optional<ExIssueLabel>[];
+  issues: Optional<ExIssue>[];
+  users: Optional<PlaneUser>[];
+  issue_comments: Optional<ExIssueComment>[];
+  cycles: Optional<ExCycle>[];
+  modules: Optional<ExModule>[];
+};
+
+export type ExIssueAttachment = {
+  id: string;
+  attributes: {
+    name: string;
+    size: number;
+  };
+  asset: string;
+  issue_id: string;
+
+  //need
+  updated_at: string;
+  updated_by: string;
+
+  external_id: string;
+  external_source: string;
+};
+
+/* ----------------- Project Type --------------------- */
+type IProject = {
+  id: string;
+  total_members: number;
+  total_cycles: number;
+  total_modules: number;
+  is_member: boolean;
+  sort_order: number;
+  member_role: number;
+  is_deployed: boolean;
+  name: string;
+  description: string;
+  description_text: any;
+  description_html: any;
+  network: number;
+  identifier: string;
+  emoji: any;
+  icon_prop: any;
+  module_view: boolean;
+  cycle_view: boolean;
+  issue_views_view: boolean;
+  page_view: boolean;
+  inbox_view: boolean;
+  is_time_tracking_enabled: boolean;
+  cover_image: string;
+  archive_in: number;
+  close_in: number;
+  logo_props: Record<string, any>;
+  archived_at: string | null;
+  start_date: string | null;
+  target_date: string | null;
+  default_assignee: any;
+  project_lead: any;
+  estimate: any;
+  default_state: any;
+};
+
+export type ExProject = Partial<IProject>;
+
+/* ----------------- Export Types --------------------- */
+export type ExIssueLabel = IIssueLabel & ExBase;
+export type ExState = IState &
+  ExBase & {
+    status: "to_be_created";
+  };
+export type ExIssue = IIsssue &
+  ExBase & {
+    links?: {
+      name: string;
+      url: string;
+    }[];
+    attachments?: ExIssueAttachment[];
+    external_source_state_id?: string;
+  };
+export type ExIssueComment = IIssueComment & ExBase;
+export type ExModule = IModule &
+  ExBase & {
+    issues: string[];
+  };
+export type ExCycle = ICycle &
+  ExBase & {
+    issues: string[];
+  };
+type ExUser = {
+  id: string;
+  first_name: string;
+  last_name: string;
+  avatar: string;
+  role: number;
+};
+
+type UserMandatePayload = {
+  email: string;
+  display_name: string;
+};
+
+export type PlaneUser = ExUser & UserMandatePayload;
+
+export type UserCreatePayload = Omit<Optional<ExUser>, "id"> &
+  UserMandatePayload & {
+    project_id: string;
+  };
+
+export type UserResponsePayload = ExUser & UserMandatePayload;

packages/sdk/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "@plane/typescript-config/base.json",
+  "compilerOptions": {
+    "baseUrl": ".",
+    "outDir": "dist",
+    "jsx": "preserve",
+    "esModuleInterop": true,
+    "paths": {
+      "@/*": ["src/*"]
+    }
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["dist", "build", "node_modules"]
+}

packages/silo/core/.eslintrc.js

@@ -0,0 +1,9 @@
+/** @type {import("eslint").Linter.Config} */
+module.exports = {
+  root: true,
+  extends: ["@plane/eslint-config/library.js"],
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    project: true,
+  },
+};

packages/silo/core/.prettierrc

@@ -0,0 +1,5 @@
+{
+  "printWidth": 120,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

packages/silo/core/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@silo/core",
+  "version": "0.23.0",
+  "private": true,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "dev": "tsc --watch",
+    "build": "tsc && tsc-alias",
+    "lint": "eslint src/**"
+  },
+  "description": "Core functionality and services shared between UI and API",
+  "author": "Plane Engineering",
+  "license": "AGPL",
+  "dependencies": {
+    "csv-string": "^4.1.1",
+    "jira.js": "^4.0.1",
+    "@plane/constants": "*",
+    "@plane/sdk": "*"
+  },
+  "devDependencies": {
+    "@plane/eslint-config": "*",
+    "@plane/typescript-config": "*",
+    "tsc-alias": "^1.8.10"
+  }
+}

packages/silo/core/src/index.ts

@@ -0,0 +1,5 @@
+// services
+export * from "./services";
+
+// types
+export * from "./types";

packages/silo/core/src/services/index.ts

@@ -0,0 +1,2 @@
+export * from "./sync-cred.service";
+export * from "./sync-job.service";

packages/silo/core/src/services/sync-cred.service.ts

@@ -0,0 +1,29 @@
+import axios, { AxiosInstance } from "axios";
+// types
+import { TSyncServiceConfigured, TSyncServices } from "@/types";
+
+export class SyncCredService {
+  public axiosInstance: AxiosInstance;
+
+  constructor(baseURL: string) {
+    this.axiosInstance = axios.create({ baseURL });
+  }
+
+  /**
+   * @description check if the service is configured
+   * @param workspaceId: string
+   * @param userId: string
+   * @param source: TSyncServices
+   * @returns TSyncServiceConfigured
+   */
+  async isServiceConfigured(
+    workspaceId: string,
+    userId: string,
+    source: TSyncServices
+  ): Promise<TSyncServiceConfigured> {
+    return this.axiosInstance
+      .get(`/silo/api/credentials/${workspaceId}/${userId}/?source=${source}`)
+      .then((response) => response?.data)
+      .catch((error) => error?.response?.data);
+  }
+}

packages/silo/core/src/services/sync-job.service.ts

@@ -0,0 +1,125 @@
+import axios, { AxiosInstance } from "axios";
+import { TSyncJobWithConfig, TSyncServices, propertiesToOmit } from "@/types";
+
+export class SyncJobService<TSyncJobConfig extends object> {
+  public axiosInstance: AxiosInstance;
+
+  constructor(baseUrl: string, xApiKey: string) {
+    this.axiosInstance = axios.create({ baseURL: baseUrl, headers: { "x-api-key": xApiKey } });
+  }
+
+  /**
+   * @description Retrieves all jobs
+   * @returns Promise resolving to an array of Job objects
+   */
+  async getSyncJobs(source: TSyncServices): Promise<TSyncJobWithConfig<TSyncJobConfig>[]> {
+    return this.axiosInstance
+      .get(`/silo/api/jobs?source=${source}`)
+      .then((res) => res.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  /**
+   * @description Fetches a job by its ID.
+   * @param jobId - Unique identifier of the job to fetch
+   * @returns Promise resolving to an array of Job objects
+   */
+  async getSyncJobById(jobId: string): Promise<TSyncJobWithConfig<TSyncJobConfig>> {
+    return this.axiosInstance
+      .get(`/silo/api/jobs/?id=${jobId}`)
+      .then((res) => res.data)
+      .catch((error) => {
+        console.log(error);
+        throw error?.response?.data;
+      });
+  }
+
+  /**
+   * @description Creates a new job.
+   * @param workspaceId - ID of the workspace
+   * @param projectId - ID of the project
+   * @param payload - Job data, excluding certain properties
+   * @returns Promise resolving to the created Job object
+   */
+  async createSyncJob(
+    workspaceId: string,
+    projectId: string,
+    payload: Omit<Partial<TSyncJobWithConfig<TSyncJobConfig>>, (typeof propertiesToOmit)[number]>
+  ) {
+    // Make workspaceId and projectId required
+    return this.axiosInstance
+      .post(`/silo/api/jobs/`, {
+        ...payload,
+        workspace_id: workspaceId,
+        project_id: projectId,
+      })
+      .then((res) => res.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  /**
+   * @description Updates an existing job.
+   * @param jobId - Unique identifier of the job to update
+   * @param payload - Partial job data to update
+   * @returns Promise resolving to the updated Job object
+   */
+  async updateSyncJob(jobId: string, payload: Partial<TSyncJobWithConfig<TSyncJobConfig>>) {
+    return this.axiosInstance
+      .put(`/silo/api/jobs/${jobId}`, payload)
+      .then((res) => res.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  /**
+   * @description Deletes a job.
+   * @param jobId - Unique identifier of the job to delete
+   * @returns Promise resolving to the deletion result
+   */
+  async deleteSyncJob(jobId: string) {
+    return this.axiosInstance
+      .delete(`/silo/api/jobs/${jobId}`)
+      .then((res) => res.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  /**
+   * @description Creates a new job configuration.
+   * @param payload - Partial job configuration data
+   * @returns Promise resolving to an object containing the inserted ID
+   */
+  async createSyncJobConfig(payload: Partial<TSyncJobConfig>): Promise<{ insertedId: string }> {
+    const configPayload = { meta: payload };
+    return this.axiosInstance
+      .post(`/silo/api/job-configs`, configPayload)
+      .then((res) => res.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+
+  /**
+   * @description Initiates a job.
+   * @param jobId - Unique identifier of the job to start
+   * @param migrationType - Type of migration, defaults to "JIRA"
+   * @returns Promise resolving to an array of Job objects
+   */
+  async startSyncJob(
+    jobId: string,
+    migrationType: TSyncServices = "JIRA"
+  ): Promise<TSyncJobWithConfig<TSyncJobConfig>[]> {
+    return this.axiosInstance
+      .post(`/silo/api/jobs/run`, { jobId, migrationType })
+      .then((res) => res.data)
+      .catch((error) => {
+        throw error?.response?.data;
+      });
+  }
+}

packages/silo/core/src/types/index.ts

@@ -0,0 +1 @@
+export * from "./root";

packages/silo/core/src/types/root.ts

@@ -0,0 +1,93 @@
+// service account types
+export type TSyncServiceCredentials = {
+  id: string;
+  source: string;
+  workspace_id: string;
+  user_id: string;
+  source_access_token: string;
+  source_refresh_token: string;
+  target_access_token: string;
+};
+
+export type TSyncServiceConfigured = {
+  isAuthenticated: boolean;
+};
+
+// importers
+export enum E_IMPORTER_KEYS {
+  JIRA = "JIRA",
+  ASANA = "ASANA",
+  LINEAR = "LINEAR",
+  TRELLO = "TRELLO",
+  GITLAB = "GITLAB",
+  SLACK = "SLACK",
+}
+export type TImporterKeys = keyof typeof E_IMPORTER_KEYS;
+
+// integrations
+export enum E_INTEGRATION_KEYS {
+  GITHUB = "GITHUB",
+}
+export type TIntegrationKeys = keyof typeof E_INTEGRATION_KEYS;
+
+// importers and integrations
+export type TSyncServices = TImporterKeys | TIntegrationKeys;
+
+export enum E_JOB_STATUS {
+  INITIATED = "INITIATED",
+  PULLING = "PULLING",
+  PULLED = "PULLED",
+  TRANSFORMING = "TRANSFORMING",
+  TRANSFORMED = "TRANSFORMED",
+  PUSHING = "PUSHING",
+  FINISHED = "FINISHED",
+  ERROR = "ERROR",
+}
+export type TSyncJobStatus = keyof typeof E_JOB_STATUS;
+
+export type TSyncJob = {
+  id: string;
+  config: string;
+  migration_type: TSyncServices;
+  project_id: string;
+  workspace_id: string;
+  workspace_slug: string;
+  credentials_id: string;
+  initiator_id: string;
+  initiator_email: string;
+  source_user_email: string;
+  source_hostname: string;
+  source_task_count: number;
+  target_hostname: string;
+  start_time?: Date;
+  end_time?: Date;
+  status: TSyncJobStatus;
+  created_at: Date;
+  updated_at: Date;
+  error: string;
+  total_batch_count: number;
+  completed_batch_count: number;
+  transformed_batch_count: number;
+};
+
+export type TSyncJobWithConfig<TSyncJobConfig = unknown> = TSyncJob & {
+  config: {
+    id: string;
+    meta: TSyncJobConfig;
+  };
+};
+
+export const propertiesToOmit = [
+  "id",
+  "config",
+  "created_at",
+  "updated_at",
+  "start_time",
+  "end_time",
+  "project_id",
+  "workspace_slug",
+] as const;
+
+export type TSyncJobConfigResponse = {
+  insertedId: string;
+};

packages/silo/core/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "@plane/typescript-config/base.json",
+  "compilerOptions": {
+    "esModuleInterop": true,
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["src/*"]
+    }
+  },
+  "include": ["src/*"],
+  "exclude": ["dist", "build", "node_modules"]
+}

packages/silo/github/.eslintrc.js

@@ -0,0 +1,36 @@
+// const { resolve } = require("node:path");
+// const project = resolve(process.cwd(), "tsconfig.json");
+
+// module.exports = {
+//   root: true,
+//   extends: ["custom"],
+//   parser: "@typescript-eslint/parser",
+//   settings: {
+//     "import/resolver": {
+//       typescript: {
+//         project,
+//       },
+//       node: {
+//         moduleDirectory: ["node_modules", "."],
+//       },
+//     },
+//   },
+//   parserOptions: {
+//     ecmaVersion: 2020,
+//     sourceType: "module",
+//     project: project,
+//   },
+//   rules: {
+//     "import/order": [
+//       "error",
+//       {
+//         groups: ["builtin", "external", "internal", "parent", "sibling"],
+//         pathGroupsExcludedImportTypes: ["builtin", "internal", "react"],
+//         alphabetize: {
+//           order: "asc",
+//           caseInsensitive: true,
+//         },
+//       },
+//     ],
+//   },
+// };

packages/silo/github/.prettierrc

@@ -0,0 +1,5 @@
+{
+  "printWidth": 120,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

packages/silo/github/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@silo/github",
+  "version": "0.23.0",
+  "private": true,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "dev": "tsc --watch",
+    "build": "tsc && tsc-alias",
+    "lint": "eslint --ext .ts src"
+  },
+  "description": "Github package used by silo for importers and integrations",
+  "author": "Plane Engineering",
+  "license": "AGPL",
+  "dependencies": {
+    "@plane/constants": "*",
+    "@plane/sdk": "*",
+    "@octokit/auth-app": "^7.1.0",
+    "@octokit/rest": "^21.0.1",
+    "axios": "^1.7.2"
+  },
+  "devDependencies": {
+    "@octokit/types": "^13.5.0",
+    "@plane/eslint-config": "*",
+    "@plane/typescript-config": "*",
+    "tsc-alias": "^1.8.10"
+  }
+}

packages/silo/github/src/etl/index.ts

@@ -0,0 +1 @@
+export * from "./pull";

packages/silo/github/src/etl/pull.ts

@@ -0,0 +1 @@
+export {};

packages/silo/github/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./services";
+export * from "./types";
+export * from "./etl";

packages/silo/github/src/services/api.service.ts

@@ -0,0 +1,97 @@
+import { createAppAuth } from "@octokit/auth-app";
+import { Octokit } from "@octokit/rest";
+
+// Service connected with octokit and facilitating github data
+export class GithubService {
+  private client: Octokit;
+
+  constructor(appId: string, privateKey: string, installationId: string) {
+    this.client = new Octokit({
+      authStrategy: createAppAuth,
+      auth: {
+        appId: appId,
+        privateKey: privateKey,
+        installationId: installationId,
+      },
+      userAgent: "octokit/rest.js v1.2.3",
+    });
+  }
+
+  async getRepos() {
+    const pageRepos = this.client.paginate.iterator(
+      this.client.apps.listReposAccessibleToInstallation,
+    );
+    const data: any = [];
+
+    for await (const { data: repos } of pageRepos) {
+      // @ts-ignore
+      data.push(...repos);
+    }
+
+    return data;
+  }
+
+  async getReposForInstallation(installationId: number) {
+    const pageRepos = this.client.paginate.iterator(
+      this.client.apps.listReposAccessibleToInstallation,
+      {
+        installation_id: installationId,
+      },
+    );
+
+    const data: any = [];
+
+    for await (const { data: repos } of pageRepos) {
+      // @ts-ignore
+      data.push(...repos);
+    }
+
+    return data;
+  }
+
+  async searchRepos(query: string) {
+    return this.client.search.repos({
+      q: query,
+    });
+  }
+
+  async getIssues(owner: string, repo: string) {
+    return this.client.issues.listForRepo({
+      owner,
+      repo,
+    });
+  }
+
+  async getLabels(owner: string, repo: string) {
+    return this.client.issues.listLabelsForRepo({
+      owner,
+      repo,
+    });
+  }
+
+  async getProjects(owner: string, repo: string) {
+    return this.client.projects.listForRepo({
+      owner,
+      repo,
+    });
+  }
+
+  async getProjectIssues(projectId: number) {
+    return this.client.projects.listColumns({
+      project_id: projectId,
+    });
+  }
+
+  async getUsersForRepo(owner: string, repo: string) {
+    return this.client.repos.listCollaborators({
+      owner,
+      repo,
+    });
+  }
+
+  async getInstallation(installationId: number) {
+    return this.client.apps.getInstallation({
+      installation_id: installationId,
+    });
+  }
+}

packages/silo/github/src/services/auth.service.ts

@@ -0,0 +1,56 @@
+import { GithubAuthConfig, GithubAuthorizeState, GithubAuthPayload, TokenResponse } from "@/types";
+import axios from "axios";
+
+export class GithubAuthService {
+  config: GithubAuthConfig;
+
+  constructor(config: GithubAuthConfig) {
+    this.config = config;
+  }
+
+  /**
+   * Generates the authorization URL for Github OAuth
+   * @param state The state object to be passed to Github
+   * @returns The full authorization URL as a string
+   */
+  getAuthUrl(state: GithubAuthorizeState): string {
+    const stateString = JSON.stringify(state);
+    return `https://github.com/apps/${this.config.appName}/installations/select_target?state=${stateString}`;
+  }
+
+  /**
+   * Exchanges the authorization code for an access token
+   * @param payload An object containing the authorization code and state
+   * @returns A promise that resolves to an object containing the token response and state
+   */
+  async getAccessToken(payload: GithubAuthPayload): Promise<{
+    response: TokenResponse;
+    state: GithubAuthorizeState;
+  }> {
+    const { code, state } = payload;
+
+    const data = {
+      code,
+      redirect_uri: this.config.callbackUrl,
+    };
+
+    const { data: response } = await axios.post(this.config.tokenUrl, data);
+    return { response, state };
+  }
+
+  /**
+   * Refreshes an existing access token using a refresh token
+   * @param refresh_token The refresh token to use for obtaining a new access token
+   * @returns A promise that resolves to the new token response
+   */
+  async getRefreshToken(refresh_token: string): Promise<TokenResponse> {
+    const data = {
+      refresh_token: refresh_token,
+      grant_type: "refresh_token",
+    };
+
+    const { data: response } = await axios.post(this.config.tokenUrl, data);
+
+    return response;
+  }
+}

packages/silo/github/src/services/index.ts

@@ -0,0 +1,2 @@
+export * from "./auth.service";
+export * from "./api.service";

packages/silo/github/src/types/index.ts

@@ -0,0 +1,36 @@
+import { RestEndpointMethodTypes } from "@octokit/rest";
+
+export type GithubEntity = {};
+
+export type GithubConfig = {};
+
+export type GithubInstallation =
+  RestEndpointMethodTypes["apps"]["getInstallation"]["response"]["data"];
+export type GithubRepository =
+  RestEndpointMethodTypes["apps"]["listReposAccessibleToInstallation"]["response"]["data"]["repositories"];
+
+export type GithubAuthPayload = {
+  code: string;
+  state: GithubAuthorizeState;
+};
+
+export type GithubAuthorizeState = {
+  workspace_slug: string;
+  workspace_id: string;
+  plane_api_token: string;
+};
+
+export type TokenResponse = {
+  access_token: string;
+  expires_in: number;
+  token_type: string;
+  scope: string;
+  refresh_token: string;
+  refresh_token_expires_in: number;
+};
+
+export type GithubAuthConfig = {
+  tokenUrl: string;
+  callbackUrl: string;
+  appName: string;
+};

packages/silo/github/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "@plane/typescript-config/base.json",
+  "compilerOptions": {
+    "esModuleInterop": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["src/*"]
+    },
+    "module": "ESNext",
+    "target": "ES6",
+    "moduleResolution": "node"
+  },
+  "include": ["src/*"],
+  "exclude": ["dist", "build", "node_modules"]
+}

packages/silo/jira/.eslintrc.js

@@ -0,0 +1,9 @@
+// /** @type {import("eslint").Linter.Config} */
+// module.exports = {
+//   root: true,
+//   extends: ["@plane/eslint-config/library.js"],
+//   parser: "@typescript-eslint/parser",
+//   parserOptions: {
+//     project: true,
+//   },
+// };

packages/silo/jira/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@silo/jira",
+  "version": "0.23.0",
+  "private": true,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "dev": "tsc --watch",
+    "build": "tsc && tsc-alias",
+    "lint": "eslint src --ext .ts"
+  },
+  "description": "Jira package used by silo for importers and integrations",
+  "author": "Plane Engineering",
+  "license": "AGPL",
+  "dependencies": {
+    "csv-string": "^4.1.1",
+    "jira.js": "^4.0.1",
+    "@plane/constants": "*",
+    "@plane/sdk": "*"
+  },
+  "devDependencies": {
+    "typescript": "^5.3.3",
+    "@plane/eslint-config": "*",
+    "@plane/typescript-config": "*",
+    "tsc-alias": "^1.8.10"
+  }
+}

packages/silo/jira/src/etl/index.ts

@@ -0,0 +1,2 @@
+export * from "./pull";
+export * from "./transform";

packages/silo/jira/src/etl/pull.ts

@@ -0,0 +1,149 @@
+import * as CSV from "csv-string";
+import {
+  Issue as IJiraIssue,
+  ComponentWithIssueCount,
+  Comment as JComment,
+} from "jira.js/out/version3/models";
+import {
+  fetchPaginatedData,
+  formatDateStringForHHMM,
+  removeArrayObjSpaces,
+} from "../helpers";
+import { JiraService } from "@/services";
+import {
+  ImportedJiraUser,
+  JiraComment,
+  JiraComponent,
+  JiraSprint,
+  PaginatedResponse,
+} from "@/types";
+
+export function pullUsers(users: string): ImportedJiraUser[] {
+  const jiraUsersObject = CSV.parse(users, { output: "objects" });
+  return removeArrayObjSpaces(jiraUsersObject) as ImportedJiraUser[];
+}
+
+export async function pullLabels(client: JiraService): Promise<string[]> {
+  const labels: string[] = [];
+  await fetchPaginatedData(
+    (startAt) => client.getResourceLabels(startAt),
+    (values) => labels.push(...(values as string[])),
+    "values"
+  );
+  return labels;
+}
+
+export async function pullIssues(
+  client: JiraService,
+  projectKey: string,
+  from?: Date
+): Promise<IJiraIssue[]> {
+  const issues: IJiraIssue[] = [];
+  await fetchPaginatedData(
+    (startAt) =>
+      client.getProjectIssues(
+        projectKey,
+        startAt,
+        from ? formatDateStringForHHMM(from) : ""
+      ),
+    (values) => issues.push(...(values as IJiraIssue[])),
+    "issues"
+  );
+  return issues;
+}
+
+export async function pullComments(
+  issues: IJiraIssue[],
+  client: JiraService
+): Promise<any[]> {
+  return await pullCommentsInBatches(issues, 20, client);
+}
+
+export async function pullSprints(
+  client: JiraService,
+  projectId: string
+): Promise<JiraSprint[]> {
+  const jiraSprints: JiraSprint[] = [];
+  try {
+    const boards = await client.getProjectBoards(projectId);
+    for (const board of boards.values) {
+      const sprints = await client.getBoardSprints(board.id as number);
+      for (const sprint of sprints.values) {
+        const boardIssues: unknown[] = [];
+        await fetchPaginatedData(
+          (startAt) =>
+            client.getBoardSprintsIssues(
+              board.id as number,
+              sprint.id as number,
+              startAt
+            ) as Promise<PaginatedResponse>,
+          (values) => boardIssues.push(...(values as IJiraIssue[])),
+          "issues"
+        );
+        jiraSprints.push({ sprint, issues: boardIssues as IJiraIssue[] });
+      }
+    }
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  } catch (e) {
+    throw new Error(`Could not fetch sprints, something went wrong`);
+  }
+  return jiraSprints;
+}
+
+export async function pullComponents(
+  client: JiraService,
+  projectKey: string
+): Promise<JiraComponent[]> {
+  const jiraComponents: JiraComponent[] = [];
+  try {
+    const jiraComponentObjects: ComponentWithIssueCount[] =
+      await client.getProjectComponents(projectKey);
+    for (const component of jiraComponentObjects) {
+      const issues = await client.getProjectComponentIssues(component.id!);
+      if (issues.issues) {
+        jiraComponents.push({ component, issues: issues.issues });
+      }
+    }
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  } catch (e) {
+    throw new Error(`Could not fetch components, something went wrong`);
+  }
+  return jiraComponents;
+}
+
+export const pullCommentsForIssue = async (
+  issue: IJiraIssue,
+  client: JiraService
+): Promise<JiraComment[]> => {
+  const comments: JiraComment[] = [];
+  await fetchPaginatedData(
+    (startAt) => client.getIssueComments(issue.id, startAt),
+    (values) => {
+      const jiraComments = values.map(
+        (comment): JiraComment => ({
+          ...(comment as JComment),
+          issue_id: issue.id,
+        })
+      );
+      comments.push(...jiraComments);
+    },
+    "comments"
+  );
+  return comments;
+};
+
+export const pullCommentsInBatches = async (
+  issues: IJiraIssue[],
+  batchSize: number,
+  client: JiraService
+): Promise<JiraComment[]> => {
+  const comments: JiraComment[] = [];
+  for (let i = 0; i < issues.length; i += batchSize) {
+    const batch = issues.slice(i, i + batchSize);
+    const batchComments = await Promise.all(
+      batch.map((issue) => pullCommentsForIssue(issue, client))
+    );
+    comments.push(...batchComments.flat());
+  }
+  return comments;
+};

packages/silo/jira/src/etl/transform.ts

@@ -0,0 +1,129 @@
+import {
+  ExCycle,
+  ExIssueComment,
+  ExIssueLabel,
+  ExModule,
+  ExIssue as PlaneIssue,
+  PlaneUser,
+} from "@plane/sdk";
+import {
+  IJiraIssue,
+  ImportedJiraUser,
+  IPriorityConfig,
+  IStateConfig,
+  JiraComment,
+  JiraComponent,
+  JiraSprint,
+} from "@/types";
+import {
+  getFormattedDate,
+  getRandomColor,
+  getTargetAttachments,
+  getTargetPriority,
+  getTargetState,
+} from "../helpers";
+
+export const transformIssue = (
+  issue: IJiraIssue,
+  resourceUrl: string,
+  stateMap: IStateConfig[],
+  priorityMap: IPriorityConfig[],
+): Partial<PlaneIssue> => {
+  const targetState = getTargetState(stateMap, issue.fields.status);
+  const targetPriority = getTargetPriority(priorityMap, issue.fields.priority);
+  const attachments = getTargetAttachments(issue.fields.attachment);
+  const renderedFields = (issue.renderedFields as { description: string }) ?? {
+    description: "<p></p>",
+  };
+  const links = [
+    {
+      name: "Linked Jira Issue",
+      url: `${resourceUrl}/browse/${issue.key}`,
+    },
+  ];
+  let description = renderedFields.description ?? "<p></p>";
+  if (description === "") {
+    description = "<p></p>";
+  }
+
+  issue.fields.labels.push("JIRA IMPORTED");
+
+  return {
+    assignees: issue.fields.assignee?.displayName
+      ? [issue.fields.assignee.displayName]
+      : [],
+    links,
+    external_id: issue.id,
+    external_source: "JIRA",
+    created_by: issue.fields.creator?.displayName,
+    name: issue.fields.summary ?? "Untitled",
+    description_html: description,
+    target_date: issue.fields.duedate,
+    start_date: issue.fields.customfield_10015,
+    created_at: issue.fields.created,
+    attachments: attachments,
+    state: targetState?.id ?? "",
+    external_source_state_id: targetState?.external_id ?? "",
+    priority: targetPriority ?? "none",
+    labels: issue.fields.labels,
+    parent: issue.fields.parent?.id,
+  } as unknown as PlaneIssue;
+};
+
+export const transformLabel = (label: string): Partial<ExIssueLabel> => {
+  return {
+    name: label,
+    color: getRandomColor(),
+  };
+};
+
+export const transformComment = (
+  comment: JiraComment,
+): Partial<ExIssueComment> => {
+  return {
+    external_id: comment.id,
+    external_source: "JIRA",
+    created_at: getFormattedDate(comment.created),
+    created_by: comment.author?.displayName,
+    comment_html: comment.renderedBody ?? "<p></p>",
+    actor: comment.author?.displayName,
+    issue: comment.issue_id,
+  };
+};
+
+export const transformUser = (user: ImportedJiraUser): Partial<PlaneUser> => {
+  const [first_name, last_name] = user.user_name.split(" ");
+  const role =
+    user.org_role && user.org_role.toLowerCase().includes("admin") ? 20 : 15;
+
+  return {
+    email: user.email,
+    display_name: user.user_name,
+    first_name: first_name ?? "",
+    last_name: last_name ?? "",
+    role,
+  };
+};
+
+export const transformSprint = (sprint: JiraSprint): Partial<ExCycle> => {
+  return {
+    external_id: sprint.sprint.id.toString(),
+    external_source: "JIRA",
+    name: sprint.sprint.name,
+    start_date: getFormattedDate(sprint.sprint.startDate),
+    end_date: getFormattedDate(sprint.sprint.endDate),
+    created_at: getFormattedDate(sprint.sprint.createdDate),
+    issues: sprint.issues.map((issue) => issue.id),
+  };
+};
+
+export const transformComponent = (
+  component: JiraComponent,
+): Partial<ExModule> => {
+  return {
+    external_id: component.component.id ?? "",
+    external_source: "JIRA",
+    name: component.component.name,
+    issues: component.issues.map((issue) => issue.id),
+  };
+};

packages/silo/jira/src/helpers/date.ts

@@ -0,0 +1,14 @@
+export const getFormattedDate = (
+  date: string | undefined
+): string | undefined => {
+  if (date) {
+    const dateObj = new Date(date);
+
+    const year = dateObj.getUTCFullYear();
+    const month = String(dateObj.getUTCMonth() + 1).padStart(2, "0"); // Months are zero-based
+    const day = String(dateObj.getUTCDate()).padStart(2, "0");
+
+    const formattedDate = `${year}-${month}-${day}`;
+    return formattedDate;
+  }
+};

packages/silo/jira/src/helpers/etl.ts

@@ -0,0 +1,83 @@
+import { IPriorityConfig, IStateConfig, PaginatedResponse } from "@/types";
+import { ExIssueAttachment, ExState } from "@plane/sdk";
+import {
+  Attachment as JiraAttachment,
+  Priority as JiraPriority,
+  StatusDetails as JiraState,
+} from "jira.js/out/version3/models";
+
+export const getTargetState = (
+  stateMap: IStateConfig[],
+  sourceState: JiraState
+): ExState | undefined => {
+  // Assign the external source and external id from jira and return the target state
+  const targetState = stateMap.find((state: IStateConfig) => {
+    if (state.source_state.id === sourceState.id) {
+      state.target_state.external_source = "JIRA";
+      state.target_state.external_id = sourceState.id as string;
+      return state;
+    }
+  });
+
+  return targetState?.target_state;
+};
+
+export const getTargetAttachments = (
+  attachments?: JiraAttachment[]
+): Partial<ExIssueAttachment[]> => {
+  if (!attachments) {
+    return [];
+  }
+  const attachmentArray = attachments
+    .map((attachment: JiraAttachment): Partial<ExIssueAttachment> => {
+      return {
+        external_id: attachment.id ?? "",
+        external_source: "JIRA",
+        attributes: {
+          name: attachment.filename ?? "Untitled",
+          size: attachment.size ?? 0,
+        },
+        asset: attachment.content ?? "",
+      };
+    })
+    .filter((attachment) => attachment !== undefined) as ExIssueAttachment[];
+
+  return attachmentArray;
+};
+
+export const getTargetPriority = (
+  priorityMap: IPriorityConfig[],
+  sourcePriority: JiraPriority
+): string | undefined => {
+  const targetPriority = priorityMap.find(
+    (priority: IPriorityConfig) =>
+      priority.source_priority.name === sourcePriority.name
+  );
+  return targetPriority?.target_priority;
+};
+
+export const fetchPaginatedData = async <T>(
+  fetchFunction: (startAt: number) => Promise<PaginatedResponse>,
+  processFunction: (values: T[]) => void,
+  listPropertyName: string
+) => {
+  let hasMore = true;
+  let startAt = 0;
+  let total = 0;
+
+  while (hasMore) {
+    const response = await fetchFunction(startAt);
+    const values = response[listPropertyName] as T[]; // Type assertion
+    if (response.total == 0) {
+      break;
+    }
+    if (response && response.total && values) {
+      total = response.total;
+      processFunction(values);
+      startAt += values.length;
+      if (response.total <= startAt) {
+        hasMore = false;
+      }
+    }
+  }
+};

packages/silo/jira/src/helpers/index.ts

@@ -0,0 +1,3 @@
+export * from "./date";
+export * from "./string";
+export * from "./etl";

packages/silo/jira/src/helpers/string.ts

@@ -0,0 +1,32 @@
+export const removeArrayObjSpaces = (arr: any[]) => {
+  return arr.map((obj) => {
+    return removeSpacesFromKeys(obj);
+  });
+};
+
+export const removeSpacesFromKeys = (obj: any) => {
+  const newObj = {};
+  for (const [key, value] of Object.entries(obj)) {
+    const newKey = key.replace(/\s+/g, "_").toLowerCase();
+    // @ts-ignore
+    newObj[newKey] = value;
+  }
+  return newObj;
+};
+
+export const formatDateStringForHHMM = (inputDate: Date): string => {
+  const date = new Date(inputDate);
+  // Extract date components
+  const year = date.getFullYear();
+  const month = (date.getMonth() + 1).toString().padStart(2, "0"); // Months are zero-based
+  const day = date.getDate().toString().padStart(2, "0");
+
+  // Construct the formatted date string
+  const formattedDate = `${year}/${month}/${day}`;
+
+  return formattedDate;
+};
+
+export const getRandomColor = () => {
+  return "#" + Math.floor(Math.random() * 16777215).toString(16);
+};

packages/silo/jira/src/index.ts

@@ -0,0 +1,4 @@
+export * from "./helpers";
+export * from "./types";
+export * from "./services";
+export * from "./etl";

packages/silo/jira/src/services/api.service.ts

@@ -0,0 +1,242 @@
+// services
+import { Version3Client } from "jira.js/out/version3";
+import axios, { AxiosError } from "axios";
+import { Board } from "jira.js/out/agile";
+import { JiraProps, JiraResource } from "@/types";
+import { PageString } from "jira.js/out/version3/models";
+
+export class JiraService {
+  private jiraClient: Version3Client;
+  private accessToken: string;
+  private refreshToken: string;
+
+  constructor(props: JiraProps) {
+    this.accessToken = props.accessToken;
+    this.jiraClient = new Version3Client({
+      host: `https://api.atlassian.com/ex/jira/${props.cloudId}`,
+      authentication: {
+        oauth2: {
+          accessToken: props.accessToken,
+        },
+      },
+    });
+
+    this.refreshToken = props.refreshToken as string;
+    this.jiraClient.handleFailedResponse = async (request) => {
+      const error = request as AxiosError;
+      if (error.response?.status === 401) {
+        try {
+          const { access_token, refresh_token, expires_in } =
+            await props.refreshTokenFunc(this.refreshToken);
+          this.refreshToken = refresh_token;
+          this.jiraClient = new Version3Client({
+            host: `https://api.atlassian.com/ex/jira/${props.cloudId}`,
+            authentication: {
+              oauth2: {
+                accessToken: access_token,
+              },
+            },
+          });
+          await props.refreshTokenCallback({
+            access_token,
+            refresh_token,
+            expires_in,
+          });
+          return request;
+        } catch (error) {
+          console.log("Error while refreshing token");
+          console.log(error);
+        }
+      }
+      throw error;
+    };
+  }
+
+  async getCurrentUser() {
+    return await this.jiraClient.myself.getCurrentUser();
+  }
+
+  async getNumberOfIssues(projectKey: string) {
+    const issues = await this.jiraClient.issueSearch.searchForIssuesUsingJql({
+      jql: `project = ${projectKey}`,
+      maxResults: 0,
+    });
+    return issues.total;
+  }
+
+  async getIssueFields() {
+    return this.jiraClient.issueFields.getFields();
+  }
+
+  async getResourceStatuses() {
+    return this.jiraClient.status.search();
+  }
+
+  // async getProjectStatuses(projectId: string) {
+  //   return this.jiraClient.status.search({
+  //     projectId: projectId,
+  //   });
+  // }
+
+  async getProjectStatuses(projectId: string) {
+    return this.jiraClient.projects.getAllStatuses({
+      projectIdOrKey: projectId,
+    });
+  }
+
+  async getFields() {
+    return this.jiraClient.issueFields.getFields();
+  }
+
+  async getProjectComponents(projectId: string) {
+    return this.jiraClient.projectComponents.getProjectComponents({
+      projectIdOrKey: projectId,
+    });
+  }
+
+  async getProjectComponentIssues(componentId: string) {
+    return this.jiraClient.issueSearch.searchForIssuesUsingJql({
+      jql: `component = ${componentId}`,
+    });
+  }
+
+  async getBoardSprints(boardId: number) {
+    const board = new Board(this.jiraClient);
+    return board.getAllSprints({
+      boardId: boardId,
+    });
+  }
+
+  async getBoardSprintsIssues(
+    boardId: number,
+    sprintId: number,
+    startAt: number,
+  ) {
+    const board = new Board(this.jiraClient);
+    return board.getBoardIssuesForSprint({
+      boardId: boardId,
+      sprintId: sprintId,
+      startAt: startAt,
+    });
+  }
+
+  async getBoardEpics(boardId: number) {
+    const board = new Board(this.jiraClient);
+    return board.getEpics({
+      boardId: boardId,
+    });
+  }
+
+  async getProjectBoards(projectId: string) {
+    const board = new Board(this.jiraClient);
+    return board.getAllBoards({
+      projectKeyOrId: projectId,
+    });
+  }
+
+  async getIssuePriorities() {
+    return this.jiraClient.issuePriorities.getPriorities();
+  }
+
+  async getResourceLabels(startAt = 0): Promise<PageString> {
+    return this.jiraClient.labels.getAllLabels({
+      startAt: startAt,
+    });
+  }
+
+  async getResourceProjects(startAt: number = 0) {
+    return this.jiraClient.projects.searchProjects({
+      startAt: startAt,
+    });
+  }
+
+  /* TODO: Confirm the endpoint */
+  async getProjectIssueTypes(projectId: string) {
+    return this.jiraClient.issueTypes.getIssueTypesForProject({
+      projectId: projectId as unknown as number,
+    });
+  }
+
+  async getProjectIssues(
+    projectKey: string,
+    startAt = 0,
+    createdAfter?: string,
+  ) {
+    return this.jiraClient.issueSearch.searchForIssuesUsingJql({
+      jql: createdAfter
+        ? `project = ${projectKey} AND (created >= "${createdAfter}" OR updated >= "${createdAfter}")`
+        : `project = ${projectKey}`,
+      expand: "renderedFields",
+      fields: ["*all"],
+      startAt,
+    });
+  }
+
+  async getAllLabels() {
+    const labels: string[] = [];
+    let startAt = 0;
+    const maxResults = 1000;
+
+    while (true) {
+      const response = await this.jiraClient.labels.getAllLabels({
+        startAt,
+        maxResults,
+      });
+
+      if (response.values) {
+        labels.push(...response.values);
+      }
+
+      if (response.isLast) {
+        break;
+      }
+
+      startAt += maxResults;
+    }
+
+    return labels;
+  }
+
+  async getNumberOfLabels(projectKey: string) {
+    const labels = await this.jiraClient.labels.getAllLabels();
+    return labels.total;
+  }
+
+  async getProjectUsers(projectKey: string) {
+    return this.jiraClient.userSearch.findAssignableUsers({
+      project: projectKey,
+    });
+  }
+
+  async getIssueComments(issueId: string, startAt: number) {
+    return await this.jiraClient.issueComments.getComments({
+      issueIdOrKey: issueId,
+      startAt: startAt,
+      expand: "renderedBody",
+    });
+  }
+
+  async getResources(): Promise<JiraResource[]> {
+    const axiosInstance = axios.create({
+      baseURL: "https://api.atlassian.com",
+    });
+
+    axiosInstance.interceptors.request.use(
+      async (config: any) => {
+        config.headers.Authorization = `Bearer ${this.accessToken}`;
+        return config;
+      },
+      (error) => {
+        return Promise.reject(error);
+      },
+    );
+
+    const response = await axiosInstance.get(
+      "/oauth/token/accessible-resources",
+    );
+
+    return response.data;
+  }
+}
+
+export default JiraService;

packages/silo/jira/src/services/auth.service.ts

@@ -0,0 +1,65 @@
+import { JiraAuthProps, JiraAuthState } from "@/types";
+import axios from "axios";
+import { JIRA_SCOPES } from "@plane/constants";
+
+export type JiraTokenResponse = {
+  access_token: string;
+  refresh_token: string;
+  expires_in: number;
+};
+
+export class JiraAuth {
+  props: JiraAuthProps;
+
+  constructor(props: JiraAuthProps) {
+    this.props = props;
+  }
+
+  getCallbackUrl(hostname: string): string {
+    // return this.props.callbackURL;
+    // remove the / at the end of the hostname
+    const host = hostname.endsWith("/") ? hostname.slice(0, -1) : hostname;
+    return host + this.props.callbackURL;
+  }
+
+  getAuthorizationURL(state: JiraAuthState, hostname: string): string {
+    const scope = JIRA_SCOPES.join(" ");
+    const callbackURL = this.getCallbackUrl(hostname);
+    const stateString = JSON.stringify(state);
+    const encodedState = Buffer.from(stateString).toString("base64");
+    const consentURL = `${this.props.authorizeURL}?client_id=${this.props.clientId}&redirect_uri=${callbackURL}&access_type=offline&response_type=code&scope=${scope}&state=${encodedState || ""}`;
+    return consentURL;
+  }
+
+  async getAccessToken(
+    code: string,
+    state: JiraAuthState,
+    hostname: string,
+  ): Promise<{ tokenResponse: JiraTokenResponse; state: JiraAuthState }> {
+    const data = {
+      code,
+      client_id: this.props.clientId,
+      client_secret: this.props.clientSecret,
+      redirect_uri: this.getCallbackUrl(hostname),
+      grant_type: "authorization_code",
+    };
+
+    const { data: tokenResponse } = await axios.post(this.props.tokenURL, data);
+    return { tokenResponse, state };
+  }
+
+  async getRefreshToken(refreshToken: string): Promise<JiraTokenResponse> {
+    const data = {
+      client_id: this.props.clientId,
+      client_secret: this.props.clientSecret,
+      refresh_token: refreshToken,
+      grant_type: "refresh_token",
+    };
+
+    const { data: response } = await axios.post(
+      "https://auth.atlassian.com/oauth/token",
+      data,
+    );
+    return response as JiraTokenResponse;
+  }
+}

packages/silo/jira/src/services/builder.ts

@@ -0,0 +1,21 @@
+import { JiraProps } from "@/types";
+import { JiraAuth } from "./auth.service";
+import JiraService from "./api.service";
+
+export const createJiraAuth = (
+  clientId: string,
+  clientSecret: string,
+  callbackURL: string,
+  authorizeURL: string,
+  tokenURL: string
+): JiraAuth =>
+  new JiraAuth({
+    clientId,
+    clientSecret,
+    callbackURL,
+    authorizeURL,
+    tokenURL,
+  });
+
+export const createJiraService = (props: JiraProps): JiraService =>
+  new JiraService(props);

packages/silo/jira/src/services/index.ts

@@ -0,0 +1,3 @@
+export * from "./builder";
+export * from "./api.service";
+export * from "./auth.service";

packages/silo/jira/src/types/index.ts

@@ -0,0 +1,151 @@
+import { ExProject, ExState } from "@plane/sdk";
+import {
+  Comment as JComment,
+  ComponentWithIssueCount,
+  Priority as JiraPriority,
+  Project as JiraProject,
+  StatusDetails as JiraStatus,
+  FieldDetails,
+  Issue,
+  IssueTypeWithStatus as JiraStates,
+} from "jira.js/out/version3/models";
+
+export type JiraProps = {
+  cloudId: string;
+  accessToken: string;
+  refreshToken: string;
+  refreshTokenFunc: (refreshToken: string) => Promise<{
+    access_token: string;
+    refresh_token: string;
+    expires_in: number;
+  }>;
+  refreshTokenCallback: (arg0: {
+    access_token: string;
+    refresh_token: string;
+    expires_in: number;
+  }) => Promise<void>;
+};
+
+export type JiraResource = {
+  id: string;
+  url: string;
+  name: string;
+  scopes: string[];
+  avatarUrl: string;
+};
+
+export type ImportedJiraUser = {
+  user_id: string;
+  user_name: string;
+  email: string;
+  user_status: string;
+  added_to_org: string;
+  org_role: string;
+};
+
+export type JiraComment = JComment & {
+  issue_id: string;
+};
+
+export type JiraSprintObject = {
+  id: number;
+  name: string;
+  state: string;
+  startDate?: string;
+  endDate?: string;
+  createdDate?: string;
+};
+
+export interface PaginatedResponse {
+  total?: number;
+  [key: string]: any; // Allow dynamic properties
+}
+
+export type JiraSprint = {
+  sprint: JiraSprintObject;
+  issues: IJiraIssue[];
+};
+
+export type JiraComponent = {
+  component: ComponentWithIssueCount;
+  issues: IJiraIssue[];
+};
+
+export type JiraEntity = {
+  labels: string[];
+  issues: IJiraIssue[];
+  users: ImportedJiraUser[];
+  issue_comments: JiraComment[];
+  sprints: JiraSprint[];
+  components: JiraComponent[];
+  customFields: FieldDetails[];
+};
+
+export interface IResource {
+  id: string;
+  url: string;
+  name: string;
+  scopes: string[];
+  avatarUrl: string;
+}
+
+// Define the type for IssueType
+export interface IIssueTypeConfig {
+  name: string;
+  value: string;
+}
+
+// Define the type for Label
+export interface ILabelConfig {
+  name: string;
+  value: boolean;
+}
+
+// Define the type for State
+export interface IStateConfig {
+  source_state: JiraStatus;
+  target_state: ExState;
+}
+
+// Define the type for Priority
+export interface IPriorityConfig {
+  source_priority: JiraPriority;
+  target_priority: string;
+}
+
+export type JiraConfig = {
+  issues: number;
+  // Users are string, as not we are saving the csv string into the config
+  users: string;
+  resource: IResource;
+  project: JiraProject;
+  planeProject: ExProject;
+  issueType: string;
+  label: ILabelConfig[];
+  state: IStateConfig[];
+  priority: IPriorityConfig[];
+};
+
+export type JiraAuthState = {
+  apiToken: string;
+  workspaceId: string;
+  workspaceSlug: string;
+  userId: string;
+};
+
+export type JiraAuthPayload = {
+  state: string;
+  code: string;
+};
+
+export type JiraAuthProps = {
+  clientId: string;
+  clientSecret: string;
+  callbackURL: string;
+  authorizeURL: string;
+  tokenURL: string;
+};
+
+export type IJiraIssue = Issue;
+
+export type { JiraProject, JiraStates, JiraStatus, JiraPriority };

packages/silo/jira/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "@plane/typescript-config/base.json",
+  "compilerOptions": {
+    "esModuleInterop": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["src/*"]
+    }
+  },
+  "include": ["src/*"],
+  "exclude": ["dist", "build", "node_modules"]
+}

packages/silo/linear/.eslintrc.js

@@ -0,0 +1,36 @@
+// const { resolve } = require("node:path");
+// const project = resolve(process.cwd(), "tsconfig.json");
+
+// module.exports = {
+//   root: true,
+//   extends: ["custom"],
+//   parser: "@typescript-eslint/parser",
+//   settings: {
+//     "import/resolver": {
+//       typescript: {
+//         project,
+//       },
+//       node: {
+//         moduleDirectory: ["node_modules", "."],
+//       },
+//     },
+//   },
+//   parserOptions: {
+//     ecmaVersion: 2020,
+//     sourceType: "module",
+//     project: project,
+//   },
+//   rules: {
+//     "import/order": [
+//       "error",
+//       {
+//         groups: ["builtin", "external", "internal", "parent", "sibling"],
+//         pathGroupsExcludedImportTypes: ["builtin", "internal", "react"],
+//         alphabetize: {
+//           order: "asc",
+//           caseInsensitive: true,
+//         },
+//       },
+//     ],
+//   },
+// };

packages/silo/linear/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@silo/linear",
+  "version": "1.0.0",
+  "repository": "https://github.com/makeplane/plane-ee",
+  "author": "Plane Engineering",
+  "license": "AGPL",
+  "private": true,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "scripts": {
+    "dev": "tsc --watch",
+    "build": "tsc && tsc-alias",
+    "lint": "eslint --ext .ts src"
+  },
+  "dependencies": {
+    "@linear/sdk": "^30.0.0",
+    "@plane/sdk": "*"
+  },
+  "devDependencies": {
+    "@plane/eslint-config": "*",
+    "@plane/typescript-config": "*",
+    "tsc-alias": "^1.8.10"
+  }
+}

packages/silo/linear/src/etl/index.ts

@@ -0,0 +1,2 @@
+export * from "./pull";
+export * from "./transform";

packages/silo/linear/src/etl/pull.ts

@@ -0,0 +1,110 @@
+import { LinearService } from "@/services";
+import { Issue, IssueLabel, User } from "@linear/sdk";
+import { LinearComment, LinearCycle, LinearIssueAttachment } from "@/types";
+
+export async function pullUsers(
+  client: LinearService,
+  teamId: string,
+): Promise<User[]> {
+  const members = await client.getTeamMembers(teamId);
+  return members.nodes;
+}
+
+export async function pullLabels(client: LinearService): Promise<IssueLabel[]> {
+  const labels = await client.getIssueLabels();
+  return labels.nodes;
+}
+
+export async function pullIssues(
+  client: LinearService,
+  teamId: string,
+): Promise<Issue[]> {
+  const issues: Issue[] = [];
+  let cursor: string | undefined;
+
+  do {
+    const response = await client.getTeamIssues(teamId, cursor);
+    issues.push(...response.nodes);
+    cursor = response.pageInfo.endCursor;
+  } while (cursor);
+
+  return issues;
+}
+
+export async function pullAttachments(
+  issues: Issue[],
+  client: LinearService,
+): Promise<LinearIssueAttachment[]> {
+  const issueIds = issues.map((issue) => issue.id);
+  const attachments = await client.getIssuesAttachments(issueIds, client);
+  return attachments;
+}
+
+export async function pullComments(
+  issues: Issue[],
+  client: LinearService,
+): Promise<LinearComment[]> {
+  const issueIds = issues.map((issue) => issue.id);
+  const comments = await client.getIssuesComments(issueIds);
+  return comments;
+}
+
+export async function pullCycles(
+  client: LinearService,
+  teamId: string,
+): Promise<LinearCycle[]> {
+  const cycles: LinearCycle[] = [];
+  try {
+    const teamCycles = await client.getTeamCycles(teamId);
+    for (const cycle of teamCycles.nodes) {
+      const cycleIssues = await client.linearClient.issues({
+        filter: {
+          cycle: { id: { eq: cycle.id } },
+          team: { id: { eq: teamId } },
+        },
+      });
+      cycles.push({ cycle, issues: cycleIssues.nodes });
+    }
+  } catch (e) {
+    throw Error(`Could not fetch cycles, something went wrong`);
+  }
+  return cycles;
+}
+
+// export const pullCommentsForIssue = async (
+//   issue: Issue,
+//   client: LinearService,
+// ): Promise<LinearComment[]> => {
+//   const comments: LinearComment[] = [];
+//   let cursor: string | undefined;
+//
+//   do {
+//     const response = await client.getIssuesComments(issues);
+//     const linearComment = response.nodes.map((comment): LinearComment => {
+//       return {
+//         ...comment,
+//         issue_id: issue.id,
+//       } as unknown as LinearComment;
+//     });
+//     comments.push(...linearComment);
+//     cursor = response.pageInfo.endCursor;
+//   } while (cursor);
+//
+//   return comments;
+// };
+//
+// export const pullCommentsInBatches = async (
+//   issues: Issue[],
+//   batchSize: number,
+//   client: LinearService,
+// ): Promise<LinearComment[]> => {
+//   const comments: LinearComment[] = [];
+//   for (let i = 0; i < issues.length; i += batchSize) {
+//     const batch = issues.slice(i, i + batchSize);
+//     const batchComments = await Promise.all(
+//       batch.map((issue) => pullCommentsForIssue(issue, client)),
+//     );
+//     comments.push(...batchComments.flat());
+//   }
+//   return comments;
+// };

packages/silo/linear/src/etl/transform.ts

@@ -0,0 +1,174 @@
+import { IStateConfig, LinearComment, LinearCycle } from "@/types";
+import {
+  ExIssue as PlaneIssue,
+  ExIssueComment,
+  PlaneUser,
+  ExCycle,
+  ExIssueAttachment,
+} from "@plane/sdk";
+import { getTargetState, getFormattedDate } from "../helpers";
+import { Issue, Comment, User, IssueLabel } from "@linear/sdk";
+
+export const transformIssue = async (
+  issue: Issue,
+  teamUrl: string,
+  users: User[],
+  labels: IssueLabel[],
+  stateMap: IStateConfig[],
+): Promise<Partial<PlaneIssue>> => {
+  let state;
+  let resolvedLabels: string[] = [];
+  await issue.assignee;
+
+  if (issue.labelIds) {
+    resolvedLabels = issue.labelIds.map((labelId) => {
+      const foundLabel = labels.find((l) => l.id === labelId);
+      return foundLabel?.name ?? "";
+    });
+  }
+
+  const assignee = await breakAndGetAssignee(issue, users);
+  const parent = await breakAndGetParent(issue);
+  const creator = await breakAndGetCreator(issue, users);
+  const targetState = state && getTargetState(stateMap, state);
+  const links = [
+    {
+      name: "Linked Linear Issue",
+      url: `${teamUrl}/issue/${issue.identifier}`,
+    },
+  ];
+
+  const attachments = extractAttachmentsFromDescription(
+    issue.description || "",
+  );
+
+  return {
+    assignees: assignee ? [assignee] : [],
+    links,
+    attachments,
+    external_id: issue.id,
+    external_source: "LINEAR",
+    created_by: creator,
+    name: issue.title,
+    description_html:
+      !issue.description || issue.description == ""
+        ? "<p></p>"
+        : issue.description,
+    target_date: getFormattedDate(issue.dueDate?.toString()),
+    start_date: getFormattedDate(issue.startedAt?.toString()),
+    created_at: issue.createdAt,
+    // state: targetState?.id ?? "",
+    // external_source_state_id: targetState?.external_id ?? "",
+    priority: issue.priority == 0 ? "none" : issue.priorityLabel.toLowerCase(),
+    labels: resolvedLabels,
+    parent: parent,
+  } as unknown as PlaneIssue;
+};
+
+export const extractAttachmentsFromDescription = (
+  description: string,
+): Partial<ExIssueAttachment>[] => {
+  const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
+  const images: Partial<ExIssueAttachment>[] = [];
+  let match;
+
+  while ((match = imageRegex.exec(description)) !== null) {
+    const [, title, url] = match;
+    // Get the last part of the url
+    const id = url.split("/").pop();
+    const attachment: Partial<ExIssueAttachment> = {
+      external_id: id ?? "",
+      external_source: "LINEAR",
+      attributes: {
+        name: title,
+        size: 0,
+      },
+      asset: url ?? "",
+    };
+
+    images.push(attachment);
+  }
+
+  return images;
+};
+
+export const transformComment = (
+  comment: LinearComment,
+  users: User[],
+): Partial<ExIssueComment> => {
+  const creator = users.find((u) => u.id === comment.user_id);
+
+  return {
+    external_id: comment.id,
+    external_source: "LINEAR",
+    created_at: getFormattedDate(comment.createdAt.toString()),
+    created_by: creator?.displayName,
+    comment_html: comment.body ?? "<p></p>",
+    actor: creator?.displayName,
+    issue: comment.issue_id,
+  };
+};
+
+export const transformUser = (user: User): Partial<PlaneUser> => {
+  const [first_name, ...lastNameParts] = user.name.split(" ");
+  const last_name = lastNameParts.join(" ");
+
+  let role = user.admin ? 20 : 15;
+
+  return {
+    email: user.email,
+    display_name: user.displayName,
+    first_name,
+    last_name,
+    role,
+  };
+};
+
+export const transformCycle = async (
+  cycle: LinearCycle,
+): Promise<Partial<ExCycle>> => {
+  return {
+    external_id: cycle.cycle.id,
+    external_source: "LINEAR",
+    name: cycle.cycle.name ?? `Cycle ${cycle.cycle.number}`,
+    start_date: getFormattedDate(cycle.cycle.startsAt.toString()),
+    end_date: getFormattedDate(cycle.cycle.endsAt.toString()),
+    created_at: getFormattedDate(cycle.cycle.createdAt.toString()),
+    issues: cycle.issues.map((issue) => issue.id),
+  };
+};
+
+const breakAndGetAssignee = async (
+  issue: Issue,
+  users: User[],
+): Promise<string | undefined> => {
+  if (issue.assignee) {
+    const assignee = await issue.assignee;
+    return assignee.displayName;
+  }
+
+  // @ts-ignore
+  const assigneeId = issue._assignee.id;
+  const user = users.find((u) => u.id === assigneeId);
+  if (user) {
+    return user.displayName;
+  }
+};
+
+const breakAndGetParent = async (issue: Issue): Promise<string | undefined> => {
+  // @ts-ignore
+  const parent = issue._parent;
+  if (parent) {
+    return parent.id;
+  }
+};
+
+const breakAndGetCreator = async (
+  issue: Issue,
+  users: User[],
+): Promise<string | undefined> => {
+  // @ts-ignore
+  const creatorId = issue._creator.id;
+  const user = users.find((u) => u.id === creatorId);
+  return user?.displayName;
+};

packages/silo/linear/src/helpers/date.helper.ts

@@ -0,0 +1,14 @@
+export const getFormattedDate = (
+  date: string | undefined
+): string | undefined => {
+  if (date) {
+    const dateObj = new Date(date);
+
+    const year = dateObj.getUTCFullYear();
+    const month = String(dateObj.getUTCMonth() + 1).padStart(2, "0"); // Months are zero-based
+    const day = String(dateObj.getUTCDate()).padStart(2, "0");
+
+    const formattedDate = `${year}-${month}-${day}`;
+    return formattedDate;
+  }
+};

packages/silo/linear/src/helpers/etl.helper.ts

@@ -0,0 +1,52 @@
+import { IPriorityConfig, IStateConfig } from "@/types";
+import { WorkflowState } from "@linear/sdk";
+import { ExIssueAttachment, ExState } from "@plane/sdk";
+
+export const getTargetState = (
+  stateMap: IStateConfig[],
+  sourceState: WorkflowState
+): ExState | undefined => {
+  const targetState = stateMap.find((state: IStateConfig) => {
+    if (state.source_state.id === sourceState.id) {
+      state.target_state.external_source = "LINEAR";
+      state.target_state.external_id = sourceState.id;
+      return state;
+    }
+  });
+
+  return targetState?.target_state;
+};
+
+export const getTargetAttachments = (
+  attachments: string[]
+): Partial<ExIssueAttachment[]> => {
+  if (!attachments) {
+    return [];
+  }
+  const attachmentArray = attachments
+    .map((attachment: string): Partial<ExIssueAttachment> => {
+      return {
+        external_id: attachment,
+        external_source: "LINEAR",
+        attributes: {
+          name: "Attachment", // Linear SDK doesn't provide attachment details, so we use a placeholder
+          size: 0,
+        },
+        asset: attachment,
+      };
+    })
+    .filter((attachment) => attachment !== undefined) as ExIssueAttachment[];
+
+  return attachmentArray;
+};
+
+export const getTargetPriority = (
+  priorityMap: IPriorityConfig[],
+  sourcePriority: number
+): string | undefined => {
+  const targetPriority = priorityMap.find(
+    (priority: IPriorityConfig) =>
+      priority.source_priority.priority === sourcePriority
+  );
+  return targetPriority?.target_priority;
+};

packages/silo/linear/src/helpers/index.ts

@@ -0,0 +1,3 @@
+export * from "./date.helper";
+export * from "./string.helper";
+export * from "./etl.helper";

packages/silo/linear/src/helpers/string.helper.ts

@@ -0,0 +1,32 @@
+export const removeArrayObjSpaces = (arr: any[]) => {
+  return arr.map((obj) => {
+    return removeSpacesFromKeys(obj);
+  });
+};
+
+export const removeSpacesFromKeys = (obj: any) => {
+  const newObj = {};
+  for (const [key, value] of Object.entries(obj)) {
+    const newKey = key.replace(/\s+/g, "_").toLowerCase();
+    // @ts-ignore
+    newObj[newKey] = value;
+  }
+  return newObj;
+};
+
+export const formatDateStringForHHMM = (inputDate: Date): string => {
+  const date = new Date(inputDate);
+  // Extract date components
+  const year = date.getFullYear();
+  const month = (date.getMonth() + 1).toString().padStart(2, "0"); // Months are zero-based
+  const day = date.getDate().toString().padStart(2, "0");
+
+  // Construct the formatted date string
+  const formattedDate = `${year}/${month}/${day}`;
+
+  return formattedDate;
+};
+
+export const getRandomColor = () => {
+  return "#" + Math.floor(Math.random() * 16777215).toString(16);
+};

packages/silo/linear/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./services";
+export * from "./types";
+export * from "./etl";

packages/silo/linear/src/services/api.service.ts

@@ -0,0 +1,178 @@
+import { Comment, LinearClient, Team, WorkflowState } from "@linear/sdk";
+import { LinearComment, LinearIssueAttachment } from "..";
+
+export type LinearProps = {
+  accessToken: string;
+};
+
+export class LinearService {
+  linearClient: LinearClient;
+
+  constructor(props: LinearProps) {
+    this.linearClient = new LinearClient({
+      accessToken: props.accessToken,
+    });
+  }
+
+  async getCurrentUser() {
+    return await this.linearClient.viewer;
+  }
+
+  async getNumberOfIssues(teamId: string) {
+    const issues = await this.linearClient.issues({
+      filter: {
+        team: { id: { eq: teamId } },
+      },
+    });
+    return issues.nodes.length;
+  }
+
+  async getIssueLabels() {
+    return await this.linearClient.issueLabels();
+  }
+
+  async getTeams() {
+    return await this.linearClient.teams();
+  }
+
+  async getTeamsWithoutPagination() {
+    const teams: Team[] = [];
+    let nextPaginateUUID: string | undefined = undefined;
+
+    while (true) {
+      const response = await this.linearClient.teams({
+        after: nextPaginateUUID,
+      });
+      if (response.nodes) {
+        teams.push(...response.nodes);
+      }
+      if (!response.pageInfo.hasNextPage) {
+        break;
+      }
+      nextPaginateUUID = response.pageInfo.endCursor;
+    }
+
+    return teams;
+  }
+
+  async getTeamStatuses(teamId: string) {
+    const team = await this.linearClient.team(teamId);
+    return await team.states();
+  }
+
+  async getTeamStatusesWithoutPagination(teamId: string) {
+    const team = await this.linearClient.team(teamId);
+    const teamStates: WorkflowState[] = [];
+    let nextPaginateUUID: string | undefined = undefined;
+
+    while (true) {
+      const response = await team.states({
+        after: nextPaginateUUID,
+      });
+      if (response.nodes) {
+        teamStates.push(...response.nodes);
+      }
+      if (!response.pageInfo.hasNextPage) {
+        break;
+      }
+      nextPaginateUUID = response.pageInfo.endCursor;
+    }
+
+    return teamStates;
+  }
+
+  async getTeamProjects(teamId: string) {
+    const team = await this.linearClient.team(teamId);
+    return await team.projects();
+  }
+
+  async getTeamIssues(teamId: string, cursor?: string) {
+    return await this.linearClient.issues({
+      first: 50,
+      after: cursor,
+      filter: {
+        team: { id: { eq: teamId } },
+      },
+    });
+  }
+
+  async getProjectIssues(projectId: string, cursor?: string) {
+    return await this.linearClient.issues({
+      first: 50,
+      after: cursor,
+      filter: {
+        project: { id: { eq: projectId } },
+      },
+    });
+  }
+
+  async getTeamMembers(teamId: string) {
+    const team = await this.linearClient.team(teamId);
+    return await team.members();
+  }
+
+  async getIssuesAttachments(
+    issues: string[],
+    client: LinearService
+  ): Promise<any> {
+    const attachments = await this.linearClient.attachments({
+      filter: {
+        title: { neq: "Original issue in Jira" },
+      },
+    });
+
+    console.log(attachments);
+  }
+
+  async getIssuesComments(issues: string[]): Promise<LinearComment[]> {
+    const comments = await this.linearClient.comments({
+      filter: {
+        issue: { id: { in: issues } },
+      },
+    });
+
+    const linearCommentPromises = comments.nodes.map(
+      async (comment): Promise<LinearComment> => {
+        const brokenIds = this.breakAndGetIds(comment);
+        return {
+          ...comment,
+          issue_id: brokenIds.issue_id,
+          user_id: brokenIds.user_id,
+        } as LinearComment;
+      }
+    );
+
+    const linearComments = (await Promise.all(
+      linearCommentPromises
+    )) as LinearComment[];
+
+    return linearComments;
+  }
+
+  async getTeamCycles(teamId: string) {
+    const team = await this.linearClient.team(teamId);
+    return await team.cycles();
+  }
+
+  async getIssuePriorities() {
+    // Linear has fixed priorities: 0 (None), 1 (Urgent), 2 (High), 3 (Medium), 4 (Low)
+    return [
+      { id: 0, name: "None" },
+      { id: 1, name: "Urgent" },
+      { id: 2, name: "High" },
+      { id: 3, name: "Medium" },
+      { id: 4, name: "Low" },
+    ];
+  }
+
+  breakAndGetIds(comment: Comment) {
+    return {
+      // @ts-ignore
+      issue_id: comment._issue.id,
+      // @ts-ignore
+      user_id: comment._user.id,
+    };
+  }
+}
+
+export default LinearService;

packages/silo/linear/src/services/auth.service.ts

@@ -0,0 +1,55 @@
+import { LinearAuthProps, LinearAuthState } from "@/types";
+import axios from "axios";
+
+export type LinearTokenResponse = {
+  access_token: string;
+  refresh_token: string;
+  expires_in: number;
+};
+
+export class LinearAuth {
+  props: LinearAuthProps;
+
+  constructor(props: LinearAuthProps) {
+    this.props = props;
+  }
+
+  getCallbackUrl(_state: LinearAuthState, hostname: string): string {
+    const host = hostname.endsWith("/") ? hostname.slice(0, -1) : hostname;
+    return host + this.props.callbackURL;
+  }
+
+  getAuthorizationURL(state: LinearAuthState, hostname: string): string {
+    const scope = "read,write"; // Linear's scope
+    const callbackURL = this.getCallbackUrl(state, hostname);
+    const stateString = JSON.stringify(state);
+    // encode state string to base64
+    const encodedState = Buffer.from(stateString).toString("base64");
+    const consentURL = `https://linear.app/oauth/authorize?client_id=${this.props.clientId}&redirect_uri=${callbackURL}&response_type=code&scope=${scope}&state=${encodedState}`;
+    return consentURL;
+  }
+
+  async getAccessToken(
+    code: string,
+    state: LinearAuthState,
+    hostname: string,
+  ): Promise<{ tokenResponse: LinearTokenResponse; state: LinearAuthState }> {
+    const params = new URLSearchParams();
+    params.append("code", code);
+    params.append("client_id", this.props.clientId);
+    params.append("client_secret", this.props.clientSecret);
+    params.append("redirect_uri", this.getCallbackUrl(state, hostname));
+    params.append("grant_type", "authorization_code");
+
+    const { data: tokenResponse } = await axios.post(
+      "https://api.linear.app/oauth/token",
+      params.toString(),
+      {
+        headers: {
+          "Content-Type": "application/x-www-form-urlencoded",
+        },
+      }
+    );
+    return { tokenResponse, state };
+  }
+}

packages/silo/linear/src/services/builder.ts

@@ -0,0 +1,8 @@
+import { LinearAuthProps } from "@/types";
+import { LinearAuth } from "./auth.service";
+import LinearService, { LinearProps } from "./api.service";
+
+export const createLinearAuthService = (props: LinearAuthProps): LinearAuth =>
+  new LinearAuth(props);
+export const createLinearService = (props: LinearProps): LinearService =>
+  new LinearService(props);

packages/silo/linear/src/services/index.ts

@@ -0,0 +1,3 @@
+export * from "./builder";
+export * from "./api.service";
+export * from "./auth.service";

packages/silo/linear/src/types/index.ts

@@ -0,0 +1,67 @@
+import {
+  Attachment,
+  Comment,
+  Cycle,
+  Issue,
+  IssueLabel,
+  IssuePriorityValue,
+  User,
+  Team,
+  WorkflowState,
+} from "@linear/sdk";
+import { ExState } from "@plane/sdk";
+
+export type LinearAuthState = {
+  workspaceId: string;
+  workspaceSlug: string;
+  apiToken: string;
+  userId: string;
+};
+
+export type LinearAuthPayload = {
+  state: string;
+  code: string;
+};
+
+export type LinearAuthProps = {
+  clientId: string;
+  clientSecret: string;
+  callbackURL: string;
+};
+
+export interface LinearConfig {
+  teamId: string;
+  teamUrl: string;
+  state: IStateConfig[];
+}
+
+export interface IStateConfig {
+  source_state: {
+    id: string;
+    name: string;
+  };
+  target_state: ExState;
+}
+
+export interface IPriorityConfig {
+  source_priority: IssuePriorityValue;
+  target_priority: string;
+}
+
+export interface LinearEntity {
+  issues: Issue[];
+  issue_comments: LinearComment[];
+  users: User[];
+  cycles: LinearCycle[];
+  labels: IssueLabel[];
+}
+
+export type LinearCycle = {
+  cycle: Cycle;
+  issues: Issue[];
+};
+
+export type LinearComment = Comment & { issue_id: string; user_id: string };
+export type LinearIssueAttachment = Attachment & { issue_id: string };
+
+export type { Team as LinearTeam, WorkflowState as LinearState };

packages/silo/linear/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "@plane/typescript-config/base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["src/*"]
+    }
+  },
+  "include": ["src/*"],
+  "exclude": ["dist", "build", "node_modules"]
+}

packages/typescript-config/express.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://json.schemastore.org/tsconfig",
+  "display": "Default",
+  "compilerOptions": {
+    "composite": false,
+    "declaration": true,
+    "declarationMap": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "inlineSources": false,
+    "isolatedModules": true,
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "preserveWatchOutput": true,
+    "skipLibCheck": true,
+    "strict": true
+  },
+  "exclude": ["node_modules"]
+}

setup.sh

@@ -10,6 +10,7 @@ cp ./apiserver/.env.example ./apiserver/.env
 cp ./space/.env.example ./space/.env
 cp ./admin/.env.example ./admin/.env
 cp ./live/.env.example ./live/.env
+cp ./silo/.env.example ./silo/.env
 
 # Generate the SECRET_KEY that will be used by django
 echo -e "\nSECRET_KEY=\"$(tr -dc 'a-z0-9' < /dev/urandom | head -c50)\"" >> ./apiserver/.env

silo/.env.example

@@ -0,0 +1,43 @@
+# App Environment Variables
+BATCH_SIZE=100
+PORT=3000
+MQ_PREFETCH_COUNT=5
+APP_BASE_URL=http://web:3000
+SILO_API_BASE_URL=http://localhost:8080
+
+# Database Settings
+POSTGRES_USER="plane"
+POSTGRES_PASSWORD="plane"
+POSTGRES_HOST="plane-db"
+POSTGRES_DB="silo"
+POSTGRES_PORT=5432
+DB_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}
+
+# RabbitMQ Environment Variables
+RABBITMQ_HOST="plane-mq"
+RABBITMQ_PORT="5672"
+RABBITMQ_USER="plane"
+RABBITMQ_PASSWORD="plane"
+RABBITMQ_VHOST="plane"
+AMQP_URL=amqp://${RABBITMQ_USER}:${RABBITMQ_PASSWORD}@${RABBITMQ_HOST}:${RABBITMQ_PORT}
+
+# Redis Environment Variables
+REDIS_HOST="plane-redis"
+REDIS_PORT="6379"
+REDIS_URL=redis://${REDIS_HOST}:${REDIS_PORT}/
+
+# Sentry Environment Variables
+SENTRY_DSN=https://examplePublicKey@o0.ingest.sentry.io/0
+
+# integrations
+# Jira Environment Variables
+JIRA_CLIENT_ID=your_jira_client_id
+JIRA_CLIENT_SECRET=your_jira_client_secret
+
+# Linear Environment Variables
+LINEAR_CLIENT_ID=your_linear_client_id
+LINEAR_CLIENT_SECRET=your_linear_client_secret
+
+# Github Environment Variables
+GITHUB_CLIENT_ID=your_github_client_id
+GITHUB_CLIENT_SECRET=your_github_client_secret

silo/.prettierignore

@@ -0,0 +1,6 @@
+node_modules
+dist
+build
+src/public
+yarn.lock
+package-lock.json

silo/.prettierrc.json

@@ -0,0 +1,5 @@
+{
+  "printWidth": 120,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

silo/Dockerfile.dev

@@ -0,0 +1,15 @@
+FROM node:18-alpine
+RUN apk add --no-cache libc6-compat
+# Set working directory
+WORKDIR /app
+
+COPY . .
+RUN yarn global add turbo
+RUN yarn install
+EXPOSE 3003
+
+ENV TURBO_TELEMETRY_DISABLED 1
+
+VOLUME [ "/app/node_modules", "/app/silo/node_modules"]
+
+CMD ["yarn","dev", "--filter=silo"]

silo/Dockerfile.silo

@@ -0,0 +1,47 @@
+FROM node:18-alpine AS base
+# The web Dockerfile is copy-pasted into our main docs at /docs/handbook/deploying-with-docker.
+# Make sure you update this Dockerfile, the Dockerfile in the web workspace and copy that over to Dockerfile in the docs.
+
+FROM base AS builder
+# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
+RUN apk update
+RUN apk add --no-cache libc6-compat
+# Set working directory
+WORKDIR /app
+RUN yarn global add turbo
+COPY . .
+RUN turbo prune silo --docker
+
+# Add lockfile and package.json's of isolated subworkspace
+FROM base AS installer
+RUN apk update
+RUN apk add --no-cache libc6-compat
+WORKDIR /app
+
+# First install dependencies (as they change less often)
+COPY .gitignore .gitignore
+COPY --from=builder /app/out/json/ .
+COPY --from=builder /app/out/yarn.lock ./yarn.lock
+RUN yarn install
+
+# Build the project and its dependencies
+COPY --from=builder /app/out/full/ .
+COPY turbo.json turbo.json
+
+ENV TURBO_TELEMETRY_DISABLED 1
+
+RUN yarn turbo build --filter=silo
+
+FROM base AS runner
+WORKDIR /app
+
+COPY --from=installer /app/silo/dist ./silo
+COPY --from=installer /app/silo/package.json ./silo/package.json
+COPY --from=installer /app/silo/src/db/migrations ./silo/src/db/migrations
+COPY --from=installer /app/silo/src/db/config ./silo/src/db/config
+COPY --from=installer /app/silo/drizzle.config.js ./silo/drizzle.config.js
+COPY --from=installer /app/node_modules ./node_modules
+
+ENV TURBO_TELEMETRY_DISABLED 1
+
+EXPOSE 3000

silo/README.md

@@ -0,0 +1 @@
+# Silo

silo/drizzle.config.js

@@ -0,0 +1,8 @@
+export default {
+  schema: "./src/db/schema/*",
+  out: "./src/db/migrations",
+  dialect: "postgresql",
+  dbCredentials: {
+    url: process.env.DB_URL || ""
+  }
+}

silo/eslint.config.mjs

@@ -0,0 +1,52 @@
+import tsParser from "@typescript-eslint/parser"
+import js from "@eslint/js"
+import { FlatCompat } from "@eslint/eslintrc"
+
+const compat = new FlatCompat({
+  recommendedConfig: js.configs.recommended,
+  allConfig: js.configs.all
+})
+
+export default [
+  ...compat.extends("eslint:recommended", "prettier"),
+  {
+    languageOptions: {
+      parser: tsParser
+    },
+    rules: {
+      "no-useless-escape": "off",
+      "prefer-const": "error",
+      "no-irregular-whitespace": "error",
+      "no-trailing-spaces": "error",
+      "no-duplicate-imports": "error",
+      "no-useless-catch": "warn",
+      "no-case-declarations": "error",
+      "no-undef": "error",
+      "no-unreachable": "error",
+      "arrow-body-style": ["error", "as-needed"],
+      "@typescript-eslint/no-unused-vars": ["error"],
+      "@typescript-eslint/no-explicit-any": "warn",
+      "@typescript-eslint/no-useless-empty-export": "error",
+      "@typescript-eslint/prefer-ts-expect-error": "error",
+      "@typescript-eslint/naming-convention": [
+        "error",
+        {
+          selector: ["function", "variable"],
+          format: ["camelCase", "snake_case", "UPPER_CASE", "PascalCase"],
+          leadingUnderscore: "allow"
+        }
+      ],
+      "import/order": [
+        "error",
+        {
+          groups: ["builtin", "external", "internal", "parent", "sibling"],
+          pathGroupsExcludedImportTypes: ["builtin", "internal", "react"],
+          alphabetize: {
+            order: "asc",
+            caseInsensitive: true
+          }
+        }
+      ]
+    }
+  }
+]

silo/nodemon.json

@@ -0,0 +1,6 @@
+{
+  "watch": ["src"],
+  "ext": "ts",
+  "ignore": ["src/public"],
+  "exec": "NODE_ENV=development ts-node -r tsconfig-paths/register src/start.ts"
+}

silo/package.json

@@ -0,0 +1,64 @@
+{
+	"name": "silo",
+	"version": "0.23.0",
+	"description": "A simple, lightweight, and fast integrations engine.",
+	"private": true,
+	"author": "engineering@plane.so",
+	"scripts": {
+		"dev": "turbo run develop",
+		"develop": "nodemon --config \"./nodemon.json\"/",
+		"build": "rm -rf ./dist/ && tsup src/start.ts --dts --minify --format cjs --out-dir dist",
+		"start": "node dist/start.js -p 8080",
+		"format": "prettier --config .prettierrc.json --write src/**/*.ts",
+		"lint": "eslint .",
+		"db:generate": "drizzle-kit generate",
+		"db:migrate": "drizzle-kit migrate"
+	},
+	"dependencies": {
+		"@sentry/node": "^8.27.0",
+		"@sentry/profiling-node": "^8.27.0",
+		"amqplib": "^0.10.4",
+		"axios": "^1.7.7",
+		"cors": "^2.8.5",
+		"csv-string": "^4.1.1",
+		"drizzle-orm": "^0.33.0",
+		"express": "^4.19.2",
+		"multer": "^1.4.5-lts.1",
+		"node-html-parser": "^6.1.13",
+		"postgres": "^3.4.4",
+		"redis": "^4.7.0",
+		"reflect-metadata": "^0.2.2",
+		"source-map-support": "^0.5.21",
+		"winston": "^3.14.2",
+		"zod": "^3.23.8"
+	},
+	"devDependencies": {
+		"ts-node": "^10.9.2",
+		"tsconfig-paths": "^4.2.0",
+		"tsup": "^8.3.0",
+		"@silo/core": "*",
+		"@silo/github": "*",
+		"@silo/jira": "*",
+		"@silo/linear": "*",
+		"@t3-oss/env-core": "^0.11.1",
+		"adm-zip": "^0.5.16",
+		"@linear/sdk": "^30.0.0",
+		"@plane/sdk": "*",
+		"@eslint/js": "^9.9.1",
+		"@plane/typescript-config": "*",
+		"@types/adm-zip": "^0.5.5",
+		"@types/amqplib": "^0.10.5",
+		"@types/cors": "^2.8.17",
+		"@types/eslint__js": "^8.42.3",
+		"@types/multer": "^1.4.12",
+		"@types/redis": "^4.0.11",
+		"drizzle-kit": "^0.22.4",
+		"eslint": "^9.9.1",
+		"nodemon": "^3.1.4",
+		"prettier": "^3.3.3",
+		"ts-to-zod": "^3.13.0",
+		"tsc-alias": "^1.8.10",
+		"typescript": "^5.3.3",
+		"typescript-eslint": "^8.4.0"
+	}
+}

silo/src/apps/engine/controllers/cred.controller.ts

@@ -0,0 +1,50 @@
+import { Controller, Get, Post } from "@/lib";
+import { createOrUpdateCredentials, getCredentialsByWorkspaceId } from "@/db/query";
+import { Request, Response } from "express";
+
+@Controller("/credentials")
+export class CredentialController {
+  @Post("/:workspaceId/:userId")
+  async upsertCredentials(req: Request, res: Response) {
+    try {
+      const workspaceId = req.params.workspaceId;
+      const userId = req.params.userId;
+
+      if (!workspaceId || !userId) {
+        return res.status(400).json({ error: "Either workspaceId or userId is not provided" });
+      }
+
+      const credential = await createOrUpdateCredentials(workspaceId, userId, {
+        workspace_id: req.params.workspaceId,
+        ...req.body,
+      });
+      res.status(200).json(credential);
+    } catch (error: any) {
+      res.status(500).json({ error: error.message });
+    }
+  }
+
+  @Get("/:workspaceId/:userId/")
+  async getCredentials(req: Request, res: Response) {
+    try {
+      // Get the workspaceId from the request params
+      const workspaceId = req.params.workspaceId;
+      const userId = req.params.userId;
+
+      if (!workspaceId || !userId) {
+        return res.status(400).json({ error: "Either workspaceId or userId is not provided" });
+      }
+
+      const source = req.query.source as string;
+      // Fetch all the credentials
+      const credentials = await getCredentialsByWorkspaceId(workspaceId, userId, source);
+
+      if (!credentials || credentials.length === 0) {
+        return res.status(401).json({ isAuthenticated: false });
+      }
+      return res.status(200).json({ isAuthenticated: true });
+    } catch (error: any) {
+      return res.status(500).json({ error: error.message });
+    }
+  }
+}

silo/src/apps/engine/controllers/index.ts

@@ -0,0 +1,2 @@
+export * from "./job.controller"
+export * from "./cred.controller"

silo/src/apps/engine/controllers/job.controller.ts

@@ -0,0 +1,195 @@
+/*
+ * A job is a fundamental unit of work in the server, a job represent a task
+ * that needs to be done from the importers or integrations present. A job and
+ * the job configuration is stored in the database and when the job is run, the
+ * same is injected inside the worker, which makes use of the job configuration
+ * and performs the task
+ */
+
+import { Controller, Delete, Get, Post, Put } from "@/lib";
+import {
+  createJob,
+  createJobConfig,
+  deleteJob,
+  getCredentialsByTargetToken,
+  getJobById,
+  getJobByWorkspaceId,
+  getJobByWorkspaceIdAndSource,
+  updateJob,
+} from "@/db/query";
+import { Request, Response } from "express";
+import taskManager from "@/apps/engine/worker";
+import { TSyncServices } from "@silo/core";
+
+@Controller("/jobs")
+export class JobController {
+  @Post("/")
+  async createJob(req: Request, res: Response) {
+    try {
+      if (!req.body.workspace_id) {
+        res.status(400).json({ message: "Workspace ID is required" });
+        return;
+      }
+      const job = await createJob({
+        ...req.body,
+      });
+      res.status(201).json(job);
+    } catch (error: any) {
+      res.status(500).json({ error: error.message });
+    }
+  }
+
+  @Get("/")
+  async getJobs(req: Request, res: Response) {
+    try {
+      // check for the params, if empty then return error
+      if (!req.query) {
+        res.status(400).json({ message: "Invalid query" });
+        return;
+      }
+      // Check for the query params and get the jobs according to the functions
+      // associated with the query params
+      if (req.query.id) {
+        const job = await getJobById(req.query.id as string);
+        res.status(200).json(job);
+        return;
+      } else {
+        // Get the api token from the headers and check if the token is valid
+        const token = req.headers["x-api-key"];
+        if (!token) {
+          res.status(401).json({ message: "Unauthorized" });
+          return;
+        }
+        // Get the credentials for the token
+        const credentials = await getCredentialsByTargetToken((token as string).trim());
+
+        if (credentials.length == 0) {
+          res.status(400).json({ message: "No migration jobs available for this token" });
+          return;
+        }
+
+        const targetCredentials = credentials[0];
+        if (targetCredentials.workspace_id == null) {
+          res.status(200).json([]);
+          return;
+        }
+        // Find the jobs based on the workspace ID of the credentials
+        let jobs = {};
+        if (req.query.source) {
+          jobs = await getJobByWorkspaceIdAndSource(targetCredentials.workspace_id, req.query.source as TSyncServices);
+        } else {
+          jobs = await getJobByWorkspaceId(targetCredentials.workspace_id);
+        }
+        res.status(200).json(jobs);
+      }
+    } catch (error: any) {
+      res.status(500).json({ error: error.message });
+    }
+  }
+
+  @Put("/:id")
+  async updateJob(req: Request, res: Response) {
+    try {
+      if (req.body.start_time) {
+        req.body.start_time = new Date(req.body.start_time);
+      }
+
+      if (req.body.end_time) {
+        req.body.end_time = new Date(req.body.end_time);
+      }
+
+      const job = await updateJob(req.params.id, req.body);
+
+      if (job) {
+        res.status(200).json(job);
+      } else {
+        res.status(404).json({ message: "Job not found" });
+      }
+    } catch (error: any) {
+      res.status(500).json({ error: error.message });
+    }
+  }
+
+  @Delete("/:id")
+  async deleteJob(req: Request, res: Response) {
+    try {
+      const job = await deleteJob(req.params.id);
+      if (job) {
+        res.status(200).json({ message: "Job deleted successfully" });
+      } else {
+        res.status(404).json({ message: "Job not found" });
+      }
+    } catch (error: any) {
+      res.status(500).json({ error: error.message });
+    }
+  }
+
+  @Post("/run")
+  async runJob(req: Request, res: Response) {
+    try {
+      const body = req.body;
+      if (
+        !body ||
+        !body.jobId ||
+        body.jobId == "" ||
+        body.jobId == null ||
+        body.migrationType == "" ||
+        body.migrationType == null
+      ) {
+        res.status(400).json({
+          message: "Invalid request, expecting (jobId) & (migrationType) to be passed",
+        });
+        return;
+      }
+
+      // Get the job from the given job id
+      const jobs = await getJobById(body.jobId);
+      if (jobs.length == 0) {
+        res.status(404).json({
+          message: `No job with id ${body.jobId} is available to run, please create one.`,
+        });
+        return;
+      }
+
+      const job = jobs[0];
+      // If the job is not finished or error, just send 400 OK, and don't do
+      // anything
+      if (job.status && job.status != "FINISHED" && job.status != "ERROR") {
+        res.status(400).json({ message: "Job already in progress, can't instantiate again" });
+        return;
+      }
+      // Check if the config is already present, for the particular job or not
+      if (!job.config || job.migration_type == null) {
+        res.status(400).json({
+          message: "Config for the requested job is not found, make sure to create a config before initiating a job",
+        });
+        return;
+      }
+
+      await taskManager.registerTask(
+        {
+          route: job.migration_type.toLowerCase(),
+          jobId: job.id,
+          type: "initiate",
+        },
+        {}
+      );
+      res.status(200).json({ message: "Job initiated successfully" });
+    } catch (error: any) {
+      res.status(500).json({ error: error.message });
+    }
+  }
+}
+
+@Controller("/job-configs")
+export class JobConfigController {
+  @Post("/")
+  async createJobConfig(req: Request, res: Response) {
+    try {
+      const jobConfig = await createJobConfig(req.body);
+      res.status(201).json(jobConfig);
+    } catch (error: any) {
+      res.status(500).json({ error: error.message });
+    }
+  }
+}

silo/src/apps/engine/worker/README.md

@@ -0,0 +1,5 @@
+## Worker
+Worker is responsible for maintaining the core, queue based import actions. The
+logical parts where the app related work is done is stored inside worker's root
+directory while the base part that will be used under the hood is stored inside
+the base directory.

silo/src/apps/engine/worker/base/consumer.ts

@@ -0,0 +1,21 @@
+import { MQActorBase } from "./mq"
+
+export class MQConsumer extends MQActorBase {
+  startConsuming(callback: (data: any) => void) {
+    return this.channel.consume(
+      this.queueName,
+      (msg: any) => {
+        if (msg && msg.content) {
+          callback(msg)
+        }
+      },
+      {
+        noAck: false
+      }
+    )
+  }
+
+  async cancelConsumer(consumer: { consumerTag: string }) {
+    await this.channel.cancel(consumer.consumerTag)
+  }
+}

silo/src/apps/engine/worker/base/index.ts

@@ -0,0 +1,3 @@
+export * from "./mq"
+export * from "./queue"
+export * from "./store"

silo/src/apps/engine/worker/base/mq.ts

@@ -0,0 +1,69 @@
+import amqp from "amqplib";
+import { TMQEntityOptions } from "./types";
+import { env } from "@/env";
+import { logger } from "@/logger";
+
+export class MQActorBase {
+  private connection!: amqp.Connection;
+  exchange: string;
+  queueName: string;
+  routingKey: string;
+  channel!: amqp.Channel;
+
+  constructor(options: TMQEntityOptions) {
+    this.exchange = "migration_exchange";
+    if (options.appType === "extension") {
+      this.queueName = options.queueName;
+      this.routingKey = options.routingKey;
+    } else {
+      this.queueName = "silo-api";
+      this.routingKey = "silo-api";
+    }
+  }
+
+  async connect() {
+    try {
+      // create connection
+      const amqpUrl = env.AMQP_URL || "amqp://localhost";
+      logger.info("Connecting to RabbitMq 🐇: ", amqpUrl);
+
+      this.connection = await amqp.connect(amqpUrl, {});
+      this.channel = await this.connection.createConfirmChannel();
+
+      // Declare the Dead Letter Exchange and Queue
+      const dlxExchange = "dlx_exchange";
+      const dlxQueue = "dlx_queue";
+
+      await this.channel.assertExchange(dlxExchange, "direct", {
+        durable: true,
+      });
+      await this.channel.assertQueue(dlxQueue, { durable: true });
+      await this.channel.bindQueue(dlxQueue, dlxExchange, "dlx_routing_key");
+
+      await this.channel.assertExchange(this.exchange, "direct", {
+        durable: true,
+      });
+
+      await this.channel.assertQueue(this.queueName, {
+        durable: true,
+        arguments: {
+          "x-dead-letter-exchange": dlxExchange,
+          "x-dead-letter-routing-key": "dlx_routing_key",
+        },
+      });
+
+      await this.channel.bindQueue(this.queueName, this.exchange, this.routingKey);
+    } catch (error) {
+      throw new Error("Error while connecting to RabbitMq: " + error);
+    }
+  }
+
+  async close() {
+    try {
+      await this.channel.close();
+      await this.connection.close();
+    } catch (error) {
+      console.log("Error while closing RabbitMq connection", error);
+    }
+  }
+}