# Document Extraction v1 > User-published content. AgentClash renders this artifact through a redacted, allowlisted public serializer. A redacted, user-published AgentClash challenge pack version. Source: https://www.agentclash.dev/publications/a29fd8e1-4412-488b-8e29-fe232397eb4c Markdown export: https://www.agentclash.dev/md/publications/a29fd8e1-4412-488b-8e29-fe232397eb4c Published: 2026-08-20T17:48:44.49283Z Updated: 2026-08-20T17:48:44.49283Z ## Challenge pack - Name: Document Extraction - Family: extraction - Description: Measures whether an agent extracts structured fields from a messy document into a strict JSON schema, with field\-level correctness checks on the precision\-critical values\. Deterministic, single\-call, reproducible\. - Version: 1 - Status: runnable ## Manifest { "challenges": [ { "category": "extraction", "definition": { "instructions": "Extract the following invoice into a single JSON object. Output ONLY the\nJSON — no prose, no code fences.\n\n--- DOCUMENT ---\nACME INDUSTRIAL SUPPLY CO.\n Invoice # INV-2024-0042\nDate: March 14, 2024\nBill To: Northwind Trading LLC\n\nQty Description Unit Price Amount\n2 Hex bolts (box) $12.50 $25.00\n1 Steel bracket $40.00 $40.00\n3 Rubber gasket $5.00 $15.00\n\nSubtotal: $80.00\nTax (8%): $6.40\nTOTAL DUE: $86.40\n--- END DOCUMENT ---\n\nReturn JSON matching this shape EXACTLY:\n\n {\n \"invoice_number\": \"\",\n \"vendor_name\": \"\",\n \"invoice_date\": \"\",\n \"currency\": \"\",\n \"total_amount\": ,\n \"line_items\": [\n { \"description\": \"\", \"quantity\": , \"unit_price\": , \"amount\": }\n ]\n }\n\nRules:\n - Normalize the date to ISO format YYYY-MM-DD.\n - currency must be the ISO 4217 code (USD), not a symbol.\n - total_amount is the TOTAL DUE as a number (no currency symbol).\n - Include every line item, in order, with numeric quantity/unit_price/amount." }, "difficulty": "medium", "instructions": "Extract the following invoice into a single JSON object. Output ONLY the\nJSON — no prose, no code fences.\n\n--- DOCUMENT ---\nACME INDUSTRIAL SUPPLY CO.\n Invoice # INV-2024-0042\nDate: March 14, 2024\nBill To: Northwind Trading LLC\n\nQty Description Unit Price Amount\n2 Hex bolts (box) $12.50 $25.00\n1 Steel bracket $40.00 $40.00\n3 Rubber gasket $5.00 $15.00\n\nSubtotal: $80.00\nTax (8%): $6.40\nTOTAL DUE: $86.40\n--- END DOCUMENT ---\n\nReturn JSON matching this shape EXACTLY:\n\n {\n \"invoice_number\": \"\",\n \"vendor_name\": \"\",\n \"invoice_date\": \"\",\n \"currency\": \"\",\n \"total_amount\": ,\n \"line_items\": [\n { \"description\": \"\", \"quantity\": , \"unit_price\": , \"amount\": }\n ]\n }\n\nRules:\n - Normalize the date to ISO format YYYY-MM-DD.\n - currency must be the ISO 4217 code (USD), not a symbol.\n - total_amount is the TOTAL DUE as a number (no currency symbol).\n - Include every line item, in order, with numeric quantity/unit_price/amount.", "key": "invoice-extraction", "title": "Extract structured fields from an invoice" } ], "evaluation_spec": { "judge_mode": "deterministic", "metrics": [ { "collector": "run_completed_successfully", "key": "completed", "type": "boolean" }, { "collector": "run_total_latency_ms", "key": "total_latency_ms", "type": "numeric", "unit": "ms" }, { "collector": "run_total_tokens", "key": "total_tokens", "type": "numeric" } ], "name": "document-extraction-v1", "pricing": {}, "runtime_limits": { "max_duration_ms": 120000, "max_total_tokens": 12000 }, "scorecard": { "dimensions": [ { "gate": true, "key": "correctness", "pass_threshold": 0.8, "source": "validators", "validators": [ "valid_extraction_schema", "invoice_number_correct", "invoice_date_normalized_iso", "total_amount_correct", "first_line_item_quantity", "third_line_item_amount" ], "weight": 0.8 }, { "key": "reliability", "source": "reliability", "weight": 0.1 }, { "better_direction": "lower", "key": "latency", "normalization": { "max": 60000, "target": 6000 }, "source": "latency", "weight": 0.05 }, { "better_direction": "lower", "key": "cost", "normalization": { "max": 0.1, "target": 0.005 }, "source": "cost", "weight": 0.05 } ], "normalization": {}, "pass_threshold": 0.7, "strategy": "weighted" }, "validators": [ { "expected_from": "literal:{\"type\":\"object\",\"required\":[\"invoice_number\",\"vendor_name\",\"invoice_date\",\"currency\",\"total_amount\",\"line_items\"],\"properties\":{\"invoice_number\":{\"type\":\"string\",\"minLength\":3},\"vendor_name\":{\"type\":\"string\",\"minLength\":2},\"invoice_date\":{\"type\":\"string\"},\"currency\":{\"type\":\"string\"},\"total_amount\":{\"type\":\"number\"},\"line_items\":{\"type\":\"array\",\"minItems\":3,\"items\":{\"type\":\"object\",\"required\":[\"description\",\"quantity\",\"unit_price\",\"amount\"],\"properties\":{\"description\":{\"type\":\"string\"},\"quantity\":{\"type\":\"integer\"},\"unit_price\":{\"type\":\"number\"},\"amount\":{\"type\":\"number\"}}}}}}", "key": "valid_extraction_schema", "target": "final_output", "type": "json_schema" }, { "expected_from": "literal:{\"path\":\"$.invoice_number\",\"comparator\":\"equals\",\"value\":\"INV-2024-0042\"}", "key": "invoice_number_correct", "target": "final_output", "type": "json_path_match" }, { "expected_from": "literal:{\"path\":\"$.invoice_date\",\"comparator\":\"equals\",\"value\":\"2024-03-14\"}", "key": "invoice_date_normalized_iso", "target": "final_output", "type": "json_path_match" }, { "expected_from": "literal:{\"path\":\"$.total_amount\",\"comparator\":\"equals\",\"value\":86.4}", "key": "total_amount_correct", "target": "final_output", "type": "json_path_match" }, { "expected_from": "literal:{\"path\":\"$.line_items[0].quantity\",\"comparator\":\"equals\",\"value\":2}", "key": "first_line_item_quantity", "target": "final_output", "type": "json_path_match" }, { "expected_from": "literal:{\"path\":\"$.line_items[2].amount\",\"comparator\":\"equals\",\"value\":15}", "key": "third_line_item_amount", "target": "final_output", "type": "json_path_match" } ], "version_number": 1 }, "filesystem": null, "input_sets": [ { "cases": [ { "case_key": "acme-invoice", "challenge_key": "invoice-extraction", "item_key": "acme-invoice", "payload": { "document_type": "invoice" } } ], "key": "default", "name": "Default Input Set" } ], "pack": { "description": "Measures whether an agent extracts structured fields from a messy document into a strict JSON schema, with field-level correctness checks on the precision-critical values. Deterministic, single-call, reproducible.", "family": "extraction", "name": "Document Extraction", "slug": "document-extraction" }, "schema_version": 1, "tool_policy": null, "version": { "assets": null, "execution_mode": "prompt_eval", "number": 1 } } ## Input sets - Default Input Set (default)