[アップデート] AWS Entity Resolution の Advanced ルールタイプがリアルタイムマッチングに対応したので試してみた

% aws s3 mb s3://er-rt-blog-123456789012 --region ap-northeast-1
make_bucket: er-rt-blog-123456789012

% aws s3 cp customers.csv s3://er-rt-blog-123456789012/input/customers.csv
upload: ./customers.csv to s3://er-rt-blog-123456789012/input/customers.csv

% aws glue create-database \
  --database-input '{"Name":"er_rt_blog"}'

% aws glue create-table \
  --database-name er_rt_blog \
  --table-input file://table-input.json

% aws glue get-table --database-name er_rt_blog --name customers \
  --query 'Table.{Name:Name,Location:StorageDescriptor.Location}'
{
    "Name": "customers",
    "Location": "s3://er-rt-blog-123456789012/input/"
}

% ROLE_ARN=$(aws iam create-role \
  --role-name er_rt_blog_role \
  --assume-role-policy-document file://trust-policy.json \
  --query 'Role.Arn' --output text)

% aws iam put-role-policy \
  --role-name er_rt_blog_role \
  --policy-name er-access \
  --policy-document file://permissions-policy.json

% echo "${ROLE_ARN}"
arn:aws:iam::123456789012:role/er_rt_blog_role

% aws entityresolution create-schema-mapping \
  --schema-name er_rt_blog_customers \
  --mapped-input-fields file://schema-fields.json
{
    "schemaName": "er_rt_blog_customers",
    "schemaArn": "arn:aws:entityresolution:ap-northeast-1:123456789012:schemamapping/er_rt_blog_customers",
    "mappedInputFields": [
        {
            "fieldName": "unique_id",
            "type": "UNIQUE_ID"
        },
        {
            "fieldName": "first_name",
            "type": "NAME_FIRST",
            "groupName": "name",
            "matchKey": "name"
        },
        {
            "fieldName": "last_name",
            "type": "NAME_LAST",
            "groupName": "name",
            "matchKey": "name"
        },
        {
            "fieldName": "email",
            "type": "EMAIL_ADDRESS",
            "matchKey": "email"
        },
        {
            "fieldName": "phone",
            "type": "PHONE_NUMBER",
            "matchKey": "phone"
        },
        {
            "fieldName": "alt_phone",
            "type": "PHONE_NUMBER",
            "matchKey": "altphone"
        }
    ]
}

% aws entityresolution create-matching-workflow \
  --workflow-name er_rt_blog_advanced \
  --input-source-config file://input-source.json \
  --output-source-config file://output-source.json \
  --resolution-techniques file://resolution-advanced.json \
  --role-arn "${ROLE_ARN}"

aws: [ERROR]: An error occurred (AccessDeniedException) when calling the CreateMatchingWorkflow operation: The service does not have access to read your data in Glue. Please check IAM role for the service to read your data.

% aws lakeformation get-data-lake-settings
{
    "DataLakeSettings": {
        "DataLakeAdmins": [
            {
                "DataLakePrincipalIdentifier": "arn:aws:iam::123456789012:role/cm-ishikawa.satoru"
            },
            {
                "DataLakePrincipalIdentifier": "arn:aws:iam::123456789012:role/service-role/AmazonSageMakerAdminIAMExecutionRole_1"
            }
        ],
        "ReadOnlyAdmins": [],
        "CreateDatabaseDefaultPermissions": [],
        "CreateTableDefaultPermissions": [],
        "Parameters": {
            "CROSS_ACCOUNT_VERSION": "1",
            "SET_CONTEXT": "FALSE"
        },
        "AllowExternalDataFiltering": false,
        "AllowFullTableExternalDataAccess": true,
        "ExternalDataFilteringAllowList": []
    }
}

% aws lakeformation grant-permissions \
  --principal "DataLakePrincipalIdentifier=${ROLE_ARN}" \
  --resource '{"Database":{"CatalogId":"123456789012","Name":"er_rt_blog"}}' \
  --permissions DESCRIBE

% aws lakeformation grant-permissions \
  --principal "DataLakePrincipalIdentifier=${ROLE_ARN}" \
  --resource '{"Table":{"CatalogId":"123456789012","DatabaseName":"er_rt_blog","Name":"customers"}}' \
  --permissions SELECT DESCRIBE

% aws entityresolution create-matching-workflow \
  --workflow-name er_rt_blog_advanced \
  --input-source-config file://input-source.json \
  --output-source-config file://output-source.json \
  --resolution-techniques file://resolution-advanced.json \
  --role-arn "${ROLE_ARN}"
{
    "workflowName": "er_rt_blog_advanced",
    "workflowArn": "arn:aws:entityresolution:ap-northeast-1:123456789012:matchingworkflow/er_rt_blog_advanced",
    "inputSourceConfig": [
        {
            "inputSourceARN": "arn:aws:glue:ap-northeast-1:123456789012:table/er_rt_blog/customers",
            "schemaName": "er_rt_blog_customers",
            "applyNormalization": true
        }
    ],
    "outputSourceConfig": [
        {
            "outputS3Path": "s3://er-rt-blog-123456789012/output/",
            "output": [
                {
                    "name": "unique_id",
                    "hashed": false
                },
                {
                    "name": "first_name",
                    "hashed": false
                },
                {
                    "name": "last_name",
                    "hashed": false
                },
                {
                    "name": "email",
                    "hashed": false
                },
                {
                    "name": "phone",
                    "hashed": false
                },
                {
                    "name": "alt_phone",
                    "hashed": false
                }
            ],
            "applyNormalization": true
        }
    ],
    "resolutionTechniques": {
        "resolutionType": "RULE_MATCHING",
        "ruleConditionProperties": {
            "rules": [
                {
                    "ruleName": "Rule1",
                    "condition": "Exact(email) AND Exact(name)"
                },
                {
                    "ruleName": "Rule2",
                    "condition": "(Exact(name) AND Exact(phone)) OR ExactManyToMany(phone, altphone)"
                }
            ]
        },
        "enableRealTimeMatching": true
    },
    "roleArn": "arn:aws:iam::123456789012:role/er_rt_blog_role"
}

% aws entityresolution generate-match-id \
  --workflow-name er_rt_blog_advanced \
  --records file://rec-rt001.json \
  --processing-type CONSISTENT
{
    "matchGroups": [
        {
            "records": [
                {
                    "inputSourceARN": "arn:aws:glue:ap-northeast-1:123456789012:table/er_rt_blog/customers",
                    "recordId": "rt-001"
                }
            ],
            "matchId": "314059b866ec4e2fa22bd6ff2bcc808b",
            "matchRule": "NoRule"
        }
    ],
    "failedRecords": []
}

% aws entityresolution generate-match-id \
  --workflow-name er_rt_blog_advanced \
  --records file://rec-rt002.json \
  --processing-type CONSISTENT
{
    "matchGroups": [
        {
            "records": [
                {
                    "inputSourceARN": "arn:aws:glue:ap-northeast-1:123456789012:table/er_rt_blog/customers",
                    "recordId": "rt-002"
                }
            ],
            "matchId": "314059b866ec4e2fa22bd6ff2bcc808b",
            "matchRule": "Rule1"
        }
    ],
    "failedRecords": []
}

% aws entityresolution generate-match-id \
  --workflow-name er_rt_blog_advanced \
  --records file://rec-rt003.json \
  --processing-type CONSISTENT
{
    "matchGroups": [
        {
            "records": [
                {
                    "inputSourceARN": "arn:aws:glue:ap-northeast-1:123456789012:table/er_rt_blog/customers",
                    "recordId": "rt-003"
                }
            ],
            "matchId": "a16dd2dc5159466994f924810321a373",
            "matchRule": "NoRule"
        }
    ],
    "failedRecords": []
}

% aws entityresolution generate-match-id \
  --workflow-name er_rt_blog_advanced \
  --records file://rec-rt004.json \
  --processing-type CONSISTENT

{
    "matchGroups": [
        {
            "records": [
                {
                    "inputSourceARN": "arn:aws:glue:ap-northeast-1:123456789012:table/er_rt_blog/customers",
                    "recordId": "rt-004"
                }
            ],
            "matchId": "a16dd2dc5159466994f924810321a373",
            "matchRule": "Rule2"
        }
    ],
    "failedRecords": []
}

% aws entityresolution generate-match-id \
  --workflow-name er_rt_blog_advanced \
  --records file://rec-rt005.json \
  --processing-type CONSISTENT

{
    "matchGroups": [
        {
            "records": [
                {
                    "inputSourceARN": "arn:aws:glue:ap-northeast-1:123456789012:table/er_rt_blog/customers",
                    "recordId": "rt-005"
                }
            ],
            "matchId": "a16dd2dc5159466994f924810321a373",
            "matchRule": "Rule2"
        }
    ],
    "failedRecords": []
}

% aws entityresolution generate-match-id \
  --workflow-name er_rt_blog_advanced \
  --records file://rec-rt006.json \
  --processing-type CONSISTENT
{
    "matchGroups": [
        {
            "records": [
                {
                    "inputSourceARN": "arn:aws:glue:ap-northeast-1:123456789012:table/er_rt_blog/customers",
                    "recordId": "rt-006"
                }
            ],
            "matchId": "de731cd48996490193443b0a78a139c0",
            "matchRule": "NoRule"
        }
    ],
    "failedRecords": []
}

% aws entityresolution generate-match-id \
  --workflow-name er_rt_blog_advanced \
  --records file://rec-rt007.json \
  --processing-type CONSISTENT
{
    "matchGroups": [
        {
            "records": [
                {
                    "inputSourceARN": "arn:aws:glue:ap-northeast-1:123456789012:table/er_rt_blog/customers",
                    "recordId": "rt-007"
                }
            ],
            "matchId": "de731cd48996490193443b0a78a139c0",
            "matchRule": "Rule2"
        }
    ],
    "failedRecords": []
}

% aws entityresolution create-matching-workflow \
  --workflow-name er_rt_blog_fuzzy \
  --input-source-config file://input-source.json \
  --output-source-config file://output-source.json \
  --resolution-techniques file://resolution-fuzzy-rt.json \
  --role-arn "${ROLE_ARN}"

aws: [ERROR]: An error occurred (ValidationException) when calling the CreateMatchingWorkflow operation: The enableRealTimeMatching parameter doesn't support fuzzy matching functions (Cosine, Levenshtein, Soundex) or EmptyValues=Ignore. Update your rules to use only exact matching functions (Exact or ExactManyToMany) without EmptyValues=Ignore.

% aws entityresolution start-matching-job --workflow-name er_rt_blog_advanced

aws: [ERROR]: An error occurred (ValidationException) when calling the StartMatchingJob operation: You can't run matching jobs on this workflow because real time matching is enabled. Create a workflow without real-time matching to run matching jobs, or use the GenerateMatchId API instead.

% aws entityresolution generate-match-id \
  --workflow-name er_rt_blog_advanced \
  --records file://rec-rt001.json \
  --processing-type EVENTUAL

aws: [ERROR]: An error occurred (ValidationException) when calling the GenerateMatchId operation: GenerateMatchId requires CONSISTENT processing type for advanced rule-based workflows. Set the processing type to CONSISTENT.

% aws entityresolution generate-match-id \
  --workflow-name er_rt_blog_advanced \
  --records file://rec-rt001.json \
  --processing-type EVENTUAL_NO_LOOKUP

aws: [ERROR]: An error occurred (ValidationException) when calling the GenerateMatchId operation: GenerateMatchId requires CONSISTENT processing type for advanced rule-based workflows. Set the processing type to CONSISTENT.

% python3 bench.py
warmup done

CONSISTENT (n=10) の実測レイテンシ [ms]
  101  107  97  108  115  91  126  78  109  70
  min=70  median=104  mean=100  max=126