[新機能]dbt Projects on Snowflakeで同一プロジェクトオブジェクトに対してdbtコマンドの並列実行が可能となりました

models/
├── core/
│   └── core_model.sql
└── marketing/
    └── marketing_model.sql

Coreスライス:
  --select path:models/core

Marketingスライス:
  --select path:models/marketing

Core       : 1時間ごと
Marketing  : 15分ごと

デプロイ単位:
  1つのdbt project object

実行単位:
  --selectで指定するモデルスライス

スケジュール単位:
  スライスごとに個別設定

snow://dbt/<database>.<schema>.<project>/versions/live

snow://dbt/ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO/versions/live

dbt実行
  ├── target/
  │   ├── manifest.json
  │   └── run_results.json
  └── logs/
      └── dbt.log
          │
          └── live versionへ書き戻し

EXECUTE DBT PROJECT ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO
  ARGS = 'build --target prod --select path:models/core'
  WRITEBACK = FALSE;

Core:
  target/core
  logs/core

Marketing:
  target/marketing
  logs/marketing

避けるべき例:

Execution A: target/
Execution B: target/marketing/

パターン1:
  WRITEBACK = FALSE

パターン2:
  WRITEBACK = TRUE
  かつ、実行ごとにtarget/log pathを分離

USE ROLE ACCOUNTADMIN;

CREATE OR REPLACE WAREHOUSE DBT_CONCURRENT_DEMO_WH
  WAREHOUSE_SIZE = XSMALL
  AUTO_SUSPEND = 60
  AUTO_RESUME = TRUE;

CREATE DATABASE IF NOT EXISTS ANALYTICS;

CREATE SCHEMA IF NOT EXISTS ANALYTICS.DBT_PROJECTS;

CREATE SCHEMA IF NOT EXISTS ANALYTICS.DBT;

CREATE ROLE DBT_CONCURRENT_DEV_ROLE;

GRANT USAGE ON WAREHOUSE DBT_CONCURRENT_DEMO_WH
  TO ROLE DBT_CONCURRENT_DEV_ROLE;

GRANT USAGE, CREATE SCHEMA ON DATABASE ANALYTICS
  TO ROLE DBT_CONCURRENT_DEV_ROLE;

GRANT USAGE, CREATE DBT PROJECT ON SCHEMA ANALYTICS.DBT_PROJECTS
  TO ROLE DBT_CONCURRENT_DEV_ROLE;

GRANT USAGE, CREATE TABLE ON SCHEMA ANALYTICS.DBT
  TO ROLE DBT_CONCURRENT_DEV_ROLE;

GRANT ROLE DBT_CONCURRENT_DEV_ROLE TO ROLE SYSADMIN; 

concurrent_demo/
├── dbt_project.yml
├── macros/
│    └── get_custom_schema.sql
└── models/
    ├── core/
    │   └── core_model.sql
    └── marketing/
        └── marketing_model.sql

`dbt_project.yml`は以下の内容にします。

```yaml
name: concurrent_demo
version: 1.0.0
config-version: 2
profile: concurrent_demo
model-paths:
  - models
analysis-paths:
  - analyses
test-paths:
  - tests
seed-paths:
  - seeds
macro-paths:
  - macros
snapshot-paths:
  - snapshots
models:
  concurrent_demo:
    core:
      +schema: DBT
      +materialized: table
    marketing:
      +schema: DBT
      +materialized: table

concurrent_demo:
  target: prod
  outputs:
    prod:
      type: snowflake
      role: DBT_CONCURRENT_DEV_ROLE
      warehouse: DBT_CONCURRENT_DEMO_WH
      database: ANALYTICS
      schema: DBT
      threads: 8

{% macro generate_schema_name(custom_schema_name, node) %}

    {% set default_schema = target.schema %}

    {# targetが「prod」かつ対象のオブジェクトが「seed」かつcustom_schemaの定義ありの場合、seed向けの「custom_schema」に #}
    {% if target.name == 'prod' and node.resource_type == 'seed' and custom_schema_name is not none %}
        {{ custom_schema_name | trim }}

    {# targetが「prod」でないかつ対象のオブジェクトが「seed」かつcustom_schemaの定義ありの場合、「default_schema」を先頭に付けたseed向けの「custom_schema」に #}
    {% elif target.name != 'prod' and node.resource_type == 'seed' and custom_schema_name is not none %}
        {{ default_schema }}_{{ custom_schema_name | trim }}

    {# targetが「prod」かつcustom_schemaの定義ありの場合、「custom_schema」に #}
    {% elif target.name == 'prod' and custom_schema_name is not none %}
        {{ custom_schema_name | trim }}

    {# custom_schemaの定義なしの場合、「default_schema」に #}
    {% elif custom_schema_name is none %}
        {{ default_schema }}

    {# 上述の条件に合致しない（targetが「prod」でないがcustom_schemaの定義あり、など）の場合、「default_schema」に #}
    {% else %}
        {{ default_schema }}
    {% endif %}

{% endmacro %}

-- models/core/core_model.sql
select
    1 as model_id,
    'core' as slice_name,
    current_timestamp() as executed_at

-- models/marketing/marketing_model.sql
select
    1 as model_id,
    'marketing' as slice_name,
    current_timestamp() as executed_at

DESCRIBE DBT PROJECT ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO;

EXECUTE DBT PROJECT ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO
  ARGS = 'build --target prod --select path:models/core'
  WRITEBACK = FALSE;

EXECUTE DBT PROJECT ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO
  ARGS = 'build --target prod --select path:models/marketing'
  WRITEBACK = FALSE;

EXECUTE DBT PROJECT ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO
  ARGS = 'build --target prod --select path:models/core --target-path target/core --log-path logs/core --log-level-file info'
  WRITEBACK = TRUE;

EXECUTE DBT PROJECT ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO
  ARGS = 'build --target prod --select path:models/marketing --target-path target/marketing --log-path logs/marketing --log-level-file info'
  WRITEBACK = TRUE;

Core:
  target/core
  logs/core

Marketing:
  target/marketing
  logs/marketing

CREATE OR REPLACE STAGE ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO_CHECK_STAGE
  ENCRYPTION = (TYPE = 'SNOWFLAKE_SSE');

COPY FILES INTO @ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO_CHECK_STAGE/core/
  FROM 'snow://dbt/ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO/versions/live/target/core/';

COPY FILES INTO @ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO_CHECK_STAGE/core/
  FROM 'snow://dbt/ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO/versions/live/logs/core/';

COPY FILES INTO @ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO_CHECK_STAGE/marketing/
  FROM 'snow://dbt/ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO/versions/live/target/marketing/';

COPY FILES INTO @ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO_CHECK_STAGE/marketing/
  FROM 'snow://dbt/ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO/versions/live/logs/marketing/';

LIST @ANALYTICS.DBT_PROJECTS.CONCURRENT_DEMO_CHECK_STAGE;