Armazenamento, acesso e fidelidade dos resultados

"""Original local scan model. Declared bytes are not BigQuery billing estimates."""
from datetime import date
import json
import re
import sys


def exact(value, keys):
    if type(value) is not dict or set(value) != set(keys):
        raise ValueError('unexpected or missing fields')


def day(value):
    if type(value) is not str or not re.fullmatch(r'[0-9]{4}-[0-9]{2}-[0-9]{2}', value):
        raise ValueError('date must be YYYY-MM-DD')
    date.fromisoformat(value)
    return value


def bounds(value):
    exact(value, ['from', 'to'])
    start, end = day(value['from']), day(value['to'])
    if start >= end:
        raise ValueError('range must be nonempty and increasing')
    return start, end


def integer(value, low, high):
    if type(value) is not int or not low <= value <= high:
        raise ValueError('integer outside local contract')


def identifier(value):
    if type(value) is not str or not re.fullmatch(r'[A-Za-z0-9_-]{1,64}', value):
        raise ValueError('invalid identifier')


def evaluate(document):
    exact(document, ['rows', 'partitionBy', 'scan', 'query'])
    rows = document['rows']
    if type(rows) is not list or len(rows) > 10000:
        raise ValueError('rows must be a list with at most 10000 entries')
    layout = document['partitionBy']
    if layout not in ('eventDate', 'ingestDate'):
        raise ValueError('unknown partition field')
    exact(document['query'], ['from', 'to', 'account'])
    query = document['query']
    start, end = bounds({'from': query['from'], 'to': query['to']})
    if query['account'] is not None:
        identifier(query['account'])
    scan = bounds(document['scan']) if document['scan'] is not None else None
    seen = set()
    for row in rows:
        exact(row, ['id', 'eventDate', 'ingestDate', 'account', 'amountCents', 'declaredBytes'])
        identifier(row['id'])
        identifier(row['account'])
        day(row['eventDate'])
        day(row['ingestDate'])
        integer(row['amountCents'], -10**12, 10**12)
        integer(row['declaredBytes'], 0, 10**9)
        if row['id'] in seen:
            raise ValueError('duplicate row id')
        seen.add(row['id'])
    def eligible(row):
        return start <= row['eventDate'] < end and (query['account'] is None or row['account'] == query['account'])
    scanned = [r for r in rows if scan is None or scan[0] <= r[layout] < scan[1]]
    reference = sorted((r for r in rows if eligible(r)), key=lambda r: r['id'])
    result = sorted((r for r in scanned if eligible(r)), key=lambda r: r['id'])
    found = {r['id'] for r in result}
    missing = [r['id'] for r in reference if r['id'] not in found]
    return {'partitionBy': layout, 'scannedPartitions': sorted({r[layout] for r in scanned}),
            'scannedRowCount': len(scanned), 'scannedDeclaredBytes': sum(r['declaredBytes'] for r in scanned),
            'referenceIds': [r['id'] for r in reference], 'resultIds': [r['id'] for r in result],
            'referenceAmountCents': sum(r['amountCents'] for r in reference),
            'resultAmountCents': sum(r['amountCents'] for r in result),
            'missingIds': missing, 'matchesProvidedReference': not missing,
            'sourceCompletenessProven': False, 'cloudPruningMeasured': False, 'billingEstimate': False}


def unique_object(pairs):
    result = {}
    for key, value in pairs:
        if key in result:
            raise ValueError('duplicate JSON field')
        result[key] = value
    return result


if __name__ == '__main__':
    try:
        raw = sys.stdin.read(3_000_001)
        if len(raw) > 3_000_000:
            raise ValueError('input exceeds local limit')
        print(json.dumps(evaluate(json.loads(raw, object_pairs_hook=unique_object)), sort_keys=True))
    except (ValueError, TypeError, RecursionError) as error:
        print(json.dumps({'error': str(error)}), file=sys.stderr)
        sys.exit(2)

Tópicos relacionados: Eventos tardios e replay · Preparação para análise · Custos e recuperação

Referência: Professional Data Engineer standard exam guide · Current linked standard guide (document title v4.2); edition date unconfirmed (2026-09-30 inspection)