dbt Job Run to Databricks Job Run

arazzo: 1.0.1
info:
  title: dbt Job Run to Databricks Job Run
  summary: Trigger a dbt Cloud transformation job, then kick off a downstream Databricks job.
  description: >-
    A data pipeline orchestration workflow that triggers a dbt Cloud job to build the
    transformed models, then starts a downstream Databricks job to consume those models
    for processing or ML. Demonstrates sequencing two compute providers so a
    transformation completes before downstream processing begins.
  version: 1.0.0
sourceDescriptions:
  - name: dbtAdminApi
    url: https://raw.githubusercontent.com/api-evangelist/dbt/refs/heads/main/openapi/dbt-cloud-administrative-api-openapi.yml
    type: openapi
  - name: databricksApi
    url: https://raw.githubusercontent.com/api-evangelist/databricks/refs/heads/main/openapi/databricks-openapi.yml
    type: openapi
workflows:
  - workflowId: dbt-job-to-databricks-run
    summary: Run a dbt job, then trigger a downstream Databricks job.
    description: >-
      Triggers a dbt Cloud job run to build the models, then runs a Databricks job to
      process the transformed output.
    inputs:
      type: object
      properties:
        accountId:
          type: integer
        jobId:
          type: integer
        databricksJobId:
          type: integer
    steps:
      - stepId: trigger-dbt
        description: Trigger the dbt Cloud transformation job run.
        operationId: $sourceDescriptions.dbtAdminApi.triggerJobRun
        parameters:
          - name: accountId
            in: path
            value: $inputs.accountId
          - name: jobId
            in: path
            value: $inputs.jobId
        requestBody:
          contentType: application/json
          payload:
            cause: Upstream transform for Databricks processing
        successCriteria:
          - condition: $statusCode == 200
        outputs:
          dbtRunId: $response.body#/data/id
      - stepId: run-databricks
        description: Trigger the downstream Databricks processing job.
        operationId: $sourceDescriptions.databricksApi.runJobNow
        requestBody:
          contentType: application/json
          payload:
            job_id: $inputs.databricksJobId
            job_parameters:
              upstream_dbt_run: $steps.trigger-dbt.outputs.dbtRunId
        successCriteria:
          - condition: $statusCode == 200
        outputs:
          databricksRunId: $response.body#/run_id
    outputs:
      dbtRunId: $steps.trigger-dbt.outputs.dbtRunId
      databricksRunId: $steps.run-databricks.outputs.databricksRunId