diff --git a/README.md b/README.md index ccf298570..1a284cb14 100644 --- a/README.md +++ b/README.md @@ -234,10 +234,11 @@ For further Information see the [license file](https://misp.github.io/misp-modul ### Action Modules * [CrowdStrike IOC Publish](https://misp.github.io/misp-modules/action_mod/#crowdstrike-ioc-publish) - Create and renew module-owned Falcon custom IOCs from MISP event and object attributes. -* [Export to Sentinel or Defender](https://misp.github.io/misp-modules/action_mod/#export-to-sentinel-or-defender) - Export indicators to Microsoft Sentinel or Microsoft Defender. Requires an existing installation of MISP2Sentinel or MISP2Defender. +* [Export to Sentinel or Defender](https://misp.github.io/misp-modules/action_mod/#export-to-sentinel-or-defender) - Export to Sentinel or Defender using the existing script wrapper, or directly to Sentinel's STIX API. * [Mattermost](https://misp.github.io/misp-modules/action_mod/#mattermost) - Simplistic module to send message to a Mattermost channel. * [Nextcloud talk](https://misp.github.io/misp-modules/action_mod/#nextcloud-talk) - Simplistic module to send a message to a Nextcloud talk conversation. * [QRadar Reference Set Publish](https://misp.github.io/misp-modules/action_mod/#qradar-reference-set-publish) - Publish typed MISP indicators to an existing QRadar reference set and explicit domain. +* [Microsoft Sentinel Publish](https://misp.github.io/misp-modules/action_mod/#microsoft-sentinel-publish) - Publish MISP indicators directly to Microsoft Sentinel's STIX upload API with per-object results. * [SentinelOne IOC Publish](https://misp.github.io/misp-modules/action_mod/#sentinelone-ioc-publish) - Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. * [Slack](https://misp.github.io/misp-modules/action_mod/#slack) - Simplistic module to send messages to a Slack channel. * [Test action](https://misp.github.io/misp-modules/action_mod/#test-action) - This module is merely a test, always returning true. Triggers on event publishing. diff --git a/documentation/README.md b/documentation/README.md index c57f6963c..970a61e0d 100644 --- a/documentation/README.md +++ b/documentation/README.md @@ -4375,6 +4375,33 @@ Create and renew module-owned Falcon custom IOCs from MISP event and object attr ----- +#### [Export to Sentinel or Defender](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/export_sentinel.py) + +Export to Sentinel or Defender using the existing script wrapper, or directly to Sentinel's STIX API. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/export_sentinel.py)] + +- **features**: +>Preserves wrapper mode and its boolean success response by default. Direct mode publishes supplied IDS attributes to Microsoft Sentinel without an external script, using the same implementation as sentinel_publish. Filtered/extracted payloads are rejected because neither path applies workflow filters. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md). + +- **config**: +>{'params': {'mode': {'type': 'string', 'description': 'wrapper (existing MISP2Sentinel/MISP2Defender) or direct (Sentinel STIX API)', 'value': 'wrapper'}, 'misp2_venv': {'type': 'string', 'description': 'MISP2Sentinel or MISP2Defender Python virtual environment', 'value': '/var/www/MISP/misp-custom/misp2sentinel/venv/'}, 'misp2_script': {'type': 'string', 'description': 'MISP2Sentinel or MISP2Defender script path', 'value': '/var/www/MISP/misp-custom/misp2sentinel/script.py'}, 'workspace_id': {'type': 'string', 'description': 'Required: target Log Analytics workspace UUID'}, 'source': {'type': 'string', 'description': 'Stable source-system namespace', 'value': 'misp-modules'}, 'expiration_days': {'type': 'string', 'description': 'Days until indicator expiry (1-365)', 'value': '30'}, 'confidence': {'type': 'string', 'description': 'STIX confidence (0-100)', 'value': '50'}, 'dry_run': {'type': 'string', 'description': 'true previews STIX payloads without network access', 'value': 'false'}}, 'blocking': False, 'support_filters': False, 'expect_misp_core_format': True} + +- **input**: +>An event UUID in wrapper mode, or supplied MISP events/object attributes in direct mode. + +- **output**: +>Boolean wrapper success, or a direct publishing summary with per-indicator outcomes. + +- **references**: +> - https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api +> - https://github.com/cudeso/misp2sentinel + +- **requirements**: +> - requests +> - MISP2Sentinel/MISP2Defender for wrapper mode, or Entra credentials for direct mode + +----- + #### [Mattermost](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/mattermost.py) Simplistic module to send message to a Mattermost channel. @@ -4428,6 +4455,32 @@ Publish typed MISP indicators to an existing QRadar reference set and explicit d ----- +#### [Microsoft Sentinel Publish](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinel_publish.py) + +Publish MISP indicators directly to Microsoft Sentinel's STIX upload API with per-object results. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinel_publish.py)] + +- **features**: +>Converts IDS-marked event and object attributes into STIX 2.1 indicators and uploads batches of up to 100 objects. Supports dry runs, explicit workspace selection, expiry and checked per-object results. Entra credentials come from the module server environment. No external script installation is required. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md). + +- **config**: +>{'params': {'workspace_id': {'type': 'string', 'description': 'Required: target Log Analytics workspace UUID'}, 'source': {'type': 'string', 'description': 'Stable source-system namespace', 'value': 'misp-modules'}, 'expiration_days': {'type': 'string', 'description': 'Days until indicator expiry (1-365)', 'value': '30'}, 'confidence': {'type': 'string', 'description': 'STIX confidence (0-100)', 'value': '50'}, 'dry_run': {'type': 'string', 'description': 'true previews STIX payloads without network access', 'value': 'false'}}, 'blocking': True, 'support_filters': False, 'expect_misp_core_format': True} + +- **input**: +>MISP events and object attributes, including attribute UUIDs and Unix timestamps. + +- **output**: +>Upload counts and per-indicator outcomes; errors signal incomplete publication. + +- **references**: +>https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api + +- **requirements**: +> - requests +> - Microsoft Sentinel Contributor at the target workspace + +----- + #### [SentinelOne IOC Publish](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinelone_ioc_publish.py) Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. diff --git a/documentation/action_mod/sentinel_publish.md b/documentation/action_mod/sentinel_publish.md new file mode 100644 index 000000000..5c3a65807 --- /dev/null +++ b/documentation/action_mod/sentinel_publish.md @@ -0,0 +1,234 @@ +# Microsoft Sentinel direct publishing + +Microsoft Sentinel indicators can now be published directly from supplied MISP +events and object attributes, without installing MISP2Sentinel or executing a +local integration script. Two action entry points share the same implementation: + +- **`sentinel_publish`** is the dedicated, blocking direct-publishing action. +- **`export_sentinel`** supports direct publishing with `mode=direct`. Its default + remains `mode=wrapper`, for existing MISP2Sentinel/MISP2Defender installations. + +Direct publishing targets Microsoft's current **STIX objects upload API**, version +`2024-02-01-preview`, in the commercial Azure cloud. Microsoft documents this API +as preview. The module uses the existing `requests` dependency; it does not use +the legacy upload-indicators endpoint or the retired Graph TI submission path. +Direct mode publishes to Sentinel; Defender publishing remains available through +the existing wrapper mode. + +## Configure direct publishing + +1. Register a Microsoft Entra application and create a client secret. Grant that + application's service principal **Microsoft Sentinel Contributor** at the + intended Log Analytics workspace. +2. Configure `SENTINEL_TENANT_ID`, `SENTINEL_CLIENT_ID` and + `SENTINEL_CLIENT_SECRET` in the environment of the **misp-modules service**, + then restart it. Tenant/client IDs must be non-nil UUIDs. Credentials are not + accepted as workflow parameters. +3. Enable **Microsoft Sentinel Publish**, or choose **Export to Sentinel or + Defender** with `mode=direct`. Supply the full intended events and objects in + workflow data and set the target workspace UUID. +4. Preview with `dry_run=true`, inspect the STIX objects and scope, then set it to + `false` to publish. Dry runs need no credentials and make no network calls; + they cannot validate Entra roles or tenant-specific API behavior. + +The workspace ID is the **Log Analytics workspace UUID**, not its Azure resource +ID, subscription ID or display name. Endpoint URLs and token audience are fixed: + +```text +POST https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token +scope = https://management.azure.com/.default + +POST https://api.ti.sentinel.azure.com/workspaces/{workspaceId}/threat-intelligence-stix-objects:upload +api-version = 2024-02-01-preview +``` + +The Entra request uses OAuth client credentials. A fresh token is obtained once +per invocation; it must have enough remaining lifetime for the publication +budget. The token and secret are never returned or included in upload payloads. +TLS verification is enabled and redirects are disabled on both endpoints. This +implementation does not support sovereign cloud endpoints, interactive login, +certificate credentials or managed identity. + +| Direct parameter | Default | Meaning | +| --- | --- | --- | +| `workspace_id` | Required | Explicit target workspace UUID. | +| `source` | `misp-modules` | Stable `sourcesystem` label, 1–64 ASCII letters, digits, spaces or `._:/-`, beginning with a letter/digit. The API reserves `Microsoft Sentinel`. | +| `expiration_days` | `30` | Integer 1–365; set each indicator's `valid_until` from the publication time. | +| `confidence` | `50` | STIX confidence, integer 0–100. | +| `dry_run` | `false` | Return planned request bodies and outcomes without authentication or upload. | + +Boolean values accept JSON booleans, integers 0/1 and strings `true`, `false`, `1`, +`0`. Empty optional values use the documented defaults. MISP displays node +defaults as placeholders; the module also applies them when values are omitted. + +## Inputs and conversion + +`data` accepts one standard (`{"Event": {...}}`), bare or legacy event envelope, +or a list of those events. Object attributes are traversed. Only supplied +`to_ids=true`, non-deleted attributes are eligible; deleted events and objects +are excluded. The direct publisher does not fetch additional MISP data. + +Every eligible attribute needs a valid **UUID** and **Unix-seconds `timestamp`**. +These are present in normal MISP core event exports and identify a source +revision. Missing, invalid or future-dated timestamps and malformed UUIDs are +counted as malformed rather than assigned unstable fallback identities. Ensure +upstream data includes those fields when supplying custom event JSON. + +| MISP type | STIX pattern path | Conversion | +| --- | --- | --- | +| `md5`, `sha1`, `sha256`, `sha512` | `file:hashes.'MD5'`, `'SHA-1'`, `'SHA-256'`, `'SHA-512'` | Validate hexadecimal length and lowercase. | +| `domain`, `hostname` | `domain-name:value` | IDNA encode, lowercase, remove one trailing DNS dot, validate labels and the 253-character normalized name limit. | +| `ip-src`, `ip-dst` | `ipv4-addr:value` or `ipv6-addr:value` | Validate and canonicalize; CIDRs and scoped IPv6 are excluded. | +| `url` | `url:value` | Require absolute HTTP(S) without credentials, whitespace or controls; retain the exact value. | +| `email`, `email-src`, `email-dst` | `email-addr:value` | Accept a bare ASCII local part and valid IDNA domain; retain local-part case. | +| `domain\|ip` | Domain plus IP | Validate both components and publish two independent indicators. | +| `filename\|md5`, `filename\|sha1`, `filename\|sha256`, `filename\|sha512`, `malware-sample` | File hash | Publish only the hash component, splitting at the last pipe. No sample or filename data is uploaded. | + +Values are quoted and escaped for STIX patterns. Unsupported types, including +filenames alone, port-qualified composites and other rule languages, are skipped. +Port constraints are not discarded to create broader indicators. Malformed values +are skipped with counts; malformed containers and configuration fail before +network access. Existing `file` and `domain-ip` objects are supported, so no new +MISP object templates are required. + +Event, object and attribute tags are combined as STIX labels. Standard TLP +white/clear, green, amber and red tags also produce the corresponding STIX TLP +marking references; clear maps to the equivalent white definition. Other handling +tags remain labels, including TLP extensions for which this implementation has no +marking-definition mapping. The destination's treatment of markings is not an +access-control guarantee: select the workspace and input according to MISP +sharing and handling policy. + +Source event/attribute/object identifiers are retained in descriptions and result +`sources`, and attributes have MISP external references. Free-text event titles, +comments and binary sample data are not copied. Duplicate representations of the +same source revision and normalized pattern are combined within an invocation. +Different MISP attribute UUIDs remain independent indicators, even if they contain +the same observable. + +## Identity and lifecycle + +A deterministic STIX `indicator--UUID` is derived from workspace, source namespace, +attribute UUID, attribute timestamp and normalized pattern. The source revision's +timestamp supplies `created` and `valid_from`; publication time supplies `modified`. +Re-running an unchanged source revision keeps its ID and creation time while +submitting a newer modification time and expiry. Changing confidence or expiry +does not change that revision's identity. + +Changing the attribute timestamp, UUID, normalized pattern, source or workspace +creates a different identity. This makes source revisions independent and avoids +changing the immutable creation timestamp of a reused STIX ID. Older revisions +are not automatically revoked, and separate attributes with equal observables +can coexist. Keep source labels stable and use the configured expiry or separate +Sentinel administration to withdraw old records. Deleted/non-IDS input is not a +deletion instruction. The module does not query existing indicators, claim +ownership of existing records, or guarantee exactly-once ingestion. + +## API example + +POST to the misp-modules `/query` endpoint for a preview: + +```json +{ + "module": "sentinel_publish", + "params": { + "workspace_id": "22549473-0938-4938-8a04-225494730938", + "source": "MISP production", + "expiration_days": "30", + "dry_run": "true" + }, + "data": { + "Event": { + "uuid": "f1ef0409-c80a-4012-b401-b7d1050f4ed4", + "Attribute": [{ + "uuid": "aa09819b-f2d8-4661-83ea-d4897cafebc8", + "timestamp": "1767225600", + "type": "domain", + "value": "example.org", + "to_ids": true + }] + } + } +} +``` + +To use the existing entry point, change `module` to `export_sentinel` and add +`"mode": "direct"` to `params`. Both direct entry points return +`result_key=sentinel_publish` and the same `data` summary: `success`, `dry_run`, +`workspace_id`, `uploaded`, `failed`, `planned`, `skipped` and `items`. Items include +the STIX ID, sources, status and sanitized failure information. Dry runs include +the exact planned API bodies under `payloads`. + +Successful workflow summaries are stored under +`_env.misp_module_results.sentinel_publish`. A top-level `error` signals failure; +MISP may then retain only the error, while standalone API callers receive the +full per-item summary. Prefer the dedicated **blocking** `sentinel_publish` action +when downstream workflow steps depend on complete publication. `export_sentinel` +keeps its existing non-blocking configuration for compatibility. + +### Upload acknowledgements and limits + +Microsoft documents a maximum of **100 objects per request** and **100 requests +per minute**. The module accepts at most **1,000 unique source-revision indicators** +per invocation and splits them into batches of up to 100. A local **1 MiB per-batch** +limit is checked before authentication; split large inputs or reduce tag metadata +if needed. Calls have a 15-second timeout and a 240-second publication budget. + +An HTTP 200 with an empty body confirms validation/publication of that batch. +HTTP 200 can also contain an `errors` array whose zero-based `recordIndex` values +identify failed STIX objects. The module validates those indices and maps failures +back to the matching items; other objects in that batch are counted as uploaded. +An empty error list is accepted. Unexpected response shapes, duplicate/out-of-range +indices, malformed JSON and non-200 responses fail without treating the batch as +successful. + +Unknown/unsuccessful batch outcomes stop subsequent batches, which are reported +as not attempted in their error messages. Indexed validation failures can coexist +with successful objects and do not prevent later batches from being submitted. +There are no automatic retries or rollbacks. **Publication can be partial.** Check +Sentinel before retrying an uncertain request. Counts describe upload API +acceptance, not immediate query visibility, newly created versus updated records, +analytics-rule matches or blocking enforcement. Provider error bodies and Entra +exception details are not exposed in results. + +## Existing wrapper compatibility + +`export_sentinel` keeps `mode=wrapper` as its default and continues to call the +configured Python executable with the event UUID as a single argument: + +```text +{misp2_venv}/bin/python {misp2_script} {event_uuid} +``` + +The existing `misp2_venv` and `misp2_script` parameters retain their default paths: +`/var/www/MISP/misp-custom/misp2sentinel/venv/` and +`/var/www/MISP/misp-custom/misp2sentinel/script.py`. Empty values now apply those +defaults. Paths must be absolute, the executable/script must exist, and the input +must identify one event with a valid UUID. The script's directory remains the +working directory. Successful wrapper calls retain `{"data": true}`. + +Wrapper failures now return `data=false` **and a top-level error**, so failures +are visible to MISP instead of appearing as a successfully executed action. The +timeout is 240 seconds to fit inside the module server's normal timeout. Script +stdout/stderr are suppressed rather than copied into service logs; configure the +integration script's own logging for diagnostics. Wrapper mode has no dry run. +Its installed integration still controls refetching, filters and remote API use. + +Both entry points advertise `support_filters=false` and reject `filteredItems` or +`matchingData` payloads. The old wrapper advertised filter support but never +applied it; it refetched the event by UUID. Select intended events upstream and +configure any wrapper-side selection in MISP2Sentinel/MISP2Defender itself. + +## Validation + +Tests validate STIX objects and patterns, current OAuth/upload wire contracts with +mocked HTTP transport, batching and indexed partial failures, credential handling, +source revision identities, objects and workflow discovery/query endpoints. +Wrapper tests cover the original argument/working-directory contract and execute +a harmless local script through a test virtual environment. No live Sentinel +workspace or Defender tenant was changed; verify credentials, workspace roles, +marking treatment and preview API availability in your deployment. + +References: [Microsoft STIX upload API](https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api), +[application/workspace setup](https://learn.microsoft.com/en-us/azure/sentinel/connect-threat-intelligence-upload-api), +and [MISP2Sentinel](https://github.com/cudeso/misp2sentinel). diff --git a/documentation/mkdocs/action_mod.md b/documentation/mkdocs/action_mod.md index 1c41ee3a1..f23001966 100644 --- a/documentation/mkdocs/action_mod.md +++ b/documentation/mkdocs/action_mod.md @@ -27,6 +27,33 @@ Create and renew module-owned Falcon custom IOCs from MISP event and object attr ----- +#### [Export to Sentinel or Defender](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/export_sentinel.py) + +Export to Sentinel or Defender using the existing script wrapper, or directly to Sentinel's STIX API. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/export_sentinel.py)] + +- **features**: +>Preserves wrapper mode and its boolean success response by default. Direct mode publishes supplied IDS attributes to Microsoft Sentinel without an external script, using the same implementation as sentinel_publish. Filtered/extracted payloads are rejected because neither path applies workflow filters. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md). + +- **config**: +>{'params': {'mode': {'type': 'string', 'description': 'wrapper (existing MISP2Sentinel/MISP2Defender) or direct (Sentinel STIX API)', 'value': 'wrapper'}, 'misp2_venv': {'type': 'string', 'description': 'MISP2Sentinel or MISP2Defender Python virtual environment', 'value': '/var/www/MISP/misp-custom/misp2sentinel/venv/'}, 'misp2_script': {'type': 'string', 'description': 'MISP2Sentinel or MISP2Defender script path', 'value': '/var/www/MISP/misp-custom/misp2sentinel/script.py'}, 'workspace_id': {'type': 'string', 'description': 'Required: target Log Analytics workspace UUID'}, 'source': {'type': 'string', 'description': 'Stable source-system namespace', 'value': 'misp-modules'}, 'expiration_days': {'type': 'string', 'description': 'Days until indicator expiry (1-365)', 'value': '30'}, 'confidence': {'type': 'string', 'description': 'STIX confidence (0-100)', 'value': '50'}, 'dry_run': {'type': 'string', 'description': 'true previews STIX payloads without network access', 'value': 'false'}}, 'blocking': False, 'support_filters': False, 'expect_misp_core_format': True} + +- **input**: +>An event UUID in wrapper mode, or supplied MISP events/object attributes in direct mode. + +- **output**: +>Boolean wrapper success, or a direct publishing summary with per-indicator outcomes. + +- **references**: +> - https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api +> - https://github.com/cudeso/misp2sentinel + +- **requirements**: +> - requests +> - MISP2Sentinel/MISP2Defender for wrapper mode, or Entra credentials for direct mode + +----- + #### [Mattermost](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/mattermost.py) Simplistic module to send message to a Mattermost channel. @@ -80,6 +107,32 @@ Publish typed MISP indicators to an existing QRadar reference set and explicit d ----- +#### [Microsoft Sentinel Publish](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinel_publish.py) + +Publish MISP indicators directly to Microsoft Sentinel's STIX upload API with per-object results. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinel_publish.py)] + +- **features**: +>Converts IDS-marked event and object attributes into STIX 2.1 indicators and uploads batches of up to 100 objects. Supports dry runs, explicit workspace selection, expiry and checked per-object results. Entra credentials come from the module server environment. No external script installation is required. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md). + +- **config**: +>{'params': {'workspace_id': {'type': 'string', 'description': 'Required: target Log Analytics workspace UUID'}, 'source': {'type': 'string', 'description': 'Stable source-system namespace', 'value': 'misp-modules'}, 'expiration_days': {'type': 'string', 'description': 'Days until indicator expiry (1-365)', 'value': '30'}, 'confidence': {'type': 'string', 'description': 'STIX confidence (0-100)', 'value': '50'}, 'dry_run': {'type': 'string', 'description': 'true previews STIX payloads without network access', 'value': 'false'}}, 'blocking': True, 'support_filters': False, 'expect_misp_core_format': True} + +- **input**: +>MISP events and object attributes, including attribute UUIDs and Unix timestamps. + +- **output**: +>Upload counts and per-indicator outcomes; errors signal incomplete publication. + +- **references**: +>https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api + +- **requirements**: +> - requests +> - Microsoft Sentinel Contributor at the target workspace + +----- + #### [SentinelOne IOC Publish](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinelone_ioc_publish.py) Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. diff --git a/documentation/mkdocs/index.md b/documentation/mkdocs/index.md index 7079156da..16bab126b 100644 --- a/documentation/mkdocs/index.md +++ b/documentation/mkdocs/index.md @@ -188,9 +188,11 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj ### Action Modules * [CrowdStrike IOC Publish](https://misp.github.io/misp-modules/action_mod/#crowdstrike-ioc-publish) - Create and renew module-owned Falcon custom IOCs from MISP event and object attributes. +* [Export to Sentinel or Defender](https://misp.github.io/misp-modules/action_mod/#export-to-sentinel-or-defender) - Export to Sentinel or Defender using the existing script wrapper, or directly to Sentinel's STIX API. * [Mattermost](https://misp.github.io/misp-modules/action_mod/#mattermost) - Simplistic module to send message to a Mattermost channel. * [Nextcloud talk](https://misp.github.io/misp-modules/action_mod/#nextcloud-talk) - Simplistic module to send a message to a Nextcloud talk conversation. * [QRadar Reference Set Publish](https://misp.github.io/misp-modules/action_mod/#qradar-reference-set-publish) - Publish typed MISP indicators to an existing QRadar reference set and explicit domain. +* [Microsoft Sentinel Publish](https://misp.github.io/misp-modules/action_mod/#microsoft-sentinel-publish) - Publish MISP indicators directly to Microsoft Sentinel's STIX upload API with per-object results. * [SentinelOne IOC Publish](https://misp.github.io/misp-modules/action_mod/#sentinelone-ioc-publish) - Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. * [Slack](https://misp.github.io/misp-modules/action_mod/#slack) - Simplistic module to send messages to a Slack channel. * [Test action](https://misp.github.io/misp-modules/action_mod/#test-action) - This module is merely a test, always returning true. Triggers on event publishing. diff --git a/misp_modules/modules/action_mod/export_sentinel.py b/misp_modules/modules/action_mod/export_sentinel.py index d479635ea..ef1eb301a 100644 --- a/misp_modules/modules/action_mod/export_sentinel.py +++ b/misp_modules/modules/action_mod/export_sentinel.py @@ -1,123 +1,138 @@ +"""Microsoft Sentinel direct publishing or the existing MISP2* script wrapper.""" + +import copy import json -import subprocess import os +import subprocess -''' - Synchronise with Microsoft Sentinel or Defender: A wrapper around an existing installation of MISP2Sentinel or MISP2Defender - - Calls the script from MISP2Sentinel or MISP2Defender (misp2_script) using the Python virtual environment (misp2_venv) - Uses the event UUID from the MISP event to export the indicators - Preferably run as Ad-Hoc workflow -''' - -misperrors = {"error": "Error"} +from misp_modules.modules.action_mod import sentinel_publish +DEFAULT_VENV = "/var/www/MISP/misp-custom/misp2sentinel/venv/" +DEFAULT_SCRIPT = "/var/www/MISP/misp-custom/misp2sentinel/script.py" moduleconfig = { "params": { - "misp2_venv": {"type": "string", "description": "Where is the MISP2Sentinel or MISP2Defender Python virtual environment?", "value": "/var/www/MISP/misp-custom/misp2sentinel/venv/"}, - "misp2_script": {"type": "string", "description": "Where is the MISP2Sentinel or MISP2Defender installation?", "value": "/var/www/MISP/misp-custom/misp2sentinel/script.py"}, + "mode": { + "type": "string", + "description": "wrapper (existing MISP2Sentinel/MISP2Defender) or direct (Sentinel STIX API)", + "value": "wrapper", + }, + "misp2_venv": { + "type": "string", + "description": "MISP2Sentinel or MISP2Defender Python virtual environment", + "value": DEFAULT_VENV, + }, + "misp2_script": { + "type": "string", + "description": "MISP2Sentinel or MISP2Defender script path", + "value": DEFAULT_SCRIPT, + }, + **copy.deepcopy(sentinel_publish.moduleconfig["params"]), }, - # Blocking modules break the exection of the current of action "blocking": False, - # Indicates whether parts of the data passed to this module should be extracted. Extracted data can be found under the `filteredItems` key - "support_filters": True, - # Indicates whether the data passed to this module should be compliant with the MISP core format - "expect_misp_core_format": False, + "support_filters": False, + "expect_misp_core_format": True, } - -returns = "boolean" - +returns = "data" moduleinfo = { - "version": "0.1", - "author": "Koen Van Impe", - "description": "Export indicators to Microsoft Sentinel or Microsoft Defender. Requires an existing installation of MISP2Sentinel or MISP2Defender.", + "version": "0.2", + "author": "Koen Van Impe, MISP Project", + "description": ( + "Export to Sentinel or Defender using the existing script wrapper, or directly to Sentinel's STIX API." + ), "module-type": ["action"], "name": "Export to Sentinel or Defender", "logo": "", - "requirements": [], - "features": "", - "references": [], - "input": "", - "output": "", + "requirements": ["requests", "MISP2Sentinel/MISP2Defender for wrapper mode, or Entra credentials for direct mode"], + "features": ( + "Preserves wrapper mode and its boolean success response by default. Direct mode publishes supplied IDS" + " attributes to Microsoft Sentinel without an external script, using the same implementation as" + " sentinel_publish. Filtered/extracted payloads are rejected because neither path applies workflow filters. See" + " the [usage" + " guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md)." + ), + "references": [ + "https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api", + "https://github.com/cudeso/misp2sentinel", + ], + "input": "An event UUID in wrapper mode, or supplied MISP events/object attributes in direct mode.", + "output": "Boolean wrapper success, or a direct publishing summary with per-indicator outcomes.", } -def export_to_sentinel(request, event_uuid): - params = request["params"] - misp2_venv = params["misp2_venv"] - misp2_script = params["misp2_script"] - - script_dir = os.path.dirname(misp2_script) - python_executable = os.path.join(misp2_venv, "bin", "python") - - if not os.path.exists(python_executable): - print(f"Error: Python executable not found at {python_executable}") - return False - - if not os.path.exists(misp2_script): - print(f"Error: Script not found at {misp2_script}") - return False +def _paths(params): + venv = params.get("misp2_venv") + script = params.get("misp2_script") + if venv is None or venv == "": + venv = DEFAULT_VENV + if script is None or script == "": + script = DEFAULT_SCRIPT + if not isinstance(venv, str) or not isinstance(script, str) or not os.path.isabs(venv) or not os.path.isabs(script): + raise ValueError("Wrapper virtual-environment and script paths must be absolute.") + if "\x00" in venv or "\x00" in script: + raise ValueError("Wrapper paths must not contain NUL characters.") + return os.path.join(venv, "bin", "python"), script - print(f"Exporting event {event_uuid} to Sentinel or Defender...") - print(f"Using Python: {python_executable}") - print(f"Running script: {misp2_script}") - print(f"Working directory: {script_dir}") +def export_to_sentinel(request, event_uuid): + """Keep the existing wrapper helper's boolean contract without logging secrets.""" try: + python, script = _paths(request.get("params", {})) + if not os.path.isfile(python) or not os.path.isfile(script): + return False result = subprocess.run( - [python_executable, misp2_script, event_uuid], - cwd=script_dir, - capture_output=True, - text=True, - timeout=300 + [python, script, event_uuid], + cwd=os.path.dirname(script), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=240, + check=False, ) - - if result.returncode == 0: - print(f"Successfully exported event {event_uuid}") - if result.stdout: - print(f"Output: {result.stdout}") - return True - else: - print(f"Export failed with return code {result.returncode}") - if result.stderr: - print(f"Error: {result.stderr}") - if result.stdout: - print(f"Output: {result.stdout}") - return False - - except subprocess.TimeoutExpired: - print("Export timed out") - return False - except Exception as e: - print(f"Error executing script: {str(e)}") + return result.returncode == 0 + except (ValueError, TypeError, OSError, subprocess.SubprocessError): return False def handler(q=False): - if q is False: + if not q: return False - - request = json.loads(q) - success = False - - event_uuid = request.get("data", {}).get("Event", {}).get("uuid") - if event_uuid: - success = export_to_sentinel(request, event_uuid) - else: - print("Error: No event UUID found in request data") - - return {"data": success} + try: + request = json.loads(q) + if not isinstance(request, dict): + raise ValueError("Request must be an object.") + if "filteredItems" in request or "matchingData" in request: + raise ValueError("Workflow filters/extracted data are unsupported; select the intended events upstream.") + params = request.get("params", {}) + if not isinstance(params, dict): + raise ValueError("params must be an object.") + mode = params.get("mode") + if mode is None or mode == "": + mode = "wrapper" + if mode == "direct": + return sentinel_publish.handler(q) + if mode != "wrapper": + raise ValueError("mode must be wrapper or direct.") + if sentinel_publish._boolean(params.get("dry_run")): + raise ValueError("dry_run is available in direct mode; wrapper mode runs the configured script.") + data = request.get("data") + if not isinstance(data, dict) or not isinstance(event := data.get("Event", data), dict): + raise ValueError("Wrapper mode requires one event; use direct mode for event collections.") + identifier = sentinel_publish._uuid(event.get("uuid"), "event UUID") + _paths(params) + except (ValueError, TypeError) as exc: + return {"data": False, "error": f"Sentinel/Defender export: {exc}"} + success = export_to_sentinel(request, identifier) + if not success: + return { + "data": False, + "error": "Sentinel/Defender wrapper failed, timed out or is unavailable; check its installation and logs.", + } + return {"data": True} def introspection(): - modulesetup = {} - try: - modulesetup["config"] = moduleconfig - except NameError: - pass - return modulesetup + return {"config": moduleconfig} def version(): - moduleinfo["config"] = moduleconfig - return moduleinfo + return dict(moduleinfo, config=moduleconfig) diff --git a/misp_modules/modules/action_mod/sentinel_publish.py b/misp_modules/modules/action_mod/sentinel_publish.py new file mode 100644 index 000000000..8e28db523 --- /dev/null +++ b/misp_modules/modules/action_mod/sentinel_publish.py @@ -0,0 +1,469 @@ +"""Publish MISP indicators directly to Microsoft Sentinel's STIX upload API.""" + +import ipaddress +import json +import os +import re +import time +import uuid +from collections import Counter +from datetime import datetime, timedelta, timezone +from urllib.parse import urlsplit + +import requests + +moduleinfo = { + "version": "1.0", + "author": "MISP Project", + "name": "Microsoft Sentinel Publish", + "description": "Publish MISP indicators directly to Microsoft Sentinel's STIX upload API with per-object results.", + "module-type": ["action"], + "logo": "", + "requirements": ["requests", "Microsoft Sentinel Contributor at the target workspace"], + "features": ( + "Converts IDS-marked event and object attributes into STIX 2.1 indicators and uploads batches of up to 100" + " objects. Supports dry runs, explicit workspace selection, expiry and checked per-object results. Entra" + " credentials come from the module server environment. No external script installation is required. See the" + " [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinel_publish.md)." + ), + "references": ["https://learn.microsoft.com/en-us/azure/sentinel/stix-objects-api"], + "input": "MISP events and object attributes, including attribute UUIDs and Unix timestamps.", + "output": "Upload counts and per-indicator outcomes; errors signal incomplete publication.", +} +moduleconfig = { + "params": { + "workspace_id": {"type": "string", "description": "Required: target Log Analytics workspace UUID"}, + "source": {"type": "string", "description": "Stable source-system namespace", "value": "misp-modules"}, + "expiration_days": {"type": "string", "description": "Days until indicator expiry (1-365)", "value": "30"}, + "confidence": {"type": "string", "description": "STIX confidence (0-100)", "value": "50"}, + "dry_run": { + "type": "string", + "description": "true previews STIX payloads without network access", + "value": "false", + }, + }, + "blocking": True, + "support_filters": False, + "expect_misp_core_format": True, +} +returns = "data" +API_VERSION = "2024-02-01-preview" +UPLOAD_ORIGIN = "https://api.ti.sentinel.azure.com" +AUTH_ORIGIN = "https://login.microsoftonline.com" +SCOPE = "https://management.azure.com/.default" +MAX_INDICATORS = 1000 +BATCH_SIZE = 100 +MAX_BATCH_BYTES = 1024 * 1024 +DEADLINE_SECONDS = 240 +HASH_LENGTHS = {"md5": ("MD5", 32), "sha1": ("SHA-1", 40), "sha256": ("SHA-256", 64), "sha512": ("SHA-512", 128)} +EMAIL_TYPES = {"email", "email-src", "email-dst"} +SUPPORTED_TYPES = ( + {"domain", "hostname", "url", "ip-src", "ip-dst", "domain|ip", "malware-sample"} + | EMAIL_TYPES + | HASH_LENGTHS.keys() + | {f"filename|{kind}" for kind in HASH_LENGTHS} +) +TLP_MARKINGS = { + "tlp:white": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + "tlp:clear": "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9", + "tlp:green": "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da", + "tlp:amber": "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82", + "tlp:red": "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed", +} + + +class PublishError(Exception): + """An error safe to return without provider bodies or credential details.""" + + +def _boolean(value): + if value is None or value == "": + return False + if isinstance(value, (bool, int)) and value in (0, 1): + return bool(value) + if isinstance(value, str) and value.lower() in ("true", "false", "1", "0"): + return value.lower() in ("true", "1") + raise ValueError("Boolean values must be true, false, 1 or 0.") + + +def _uuid(value, name): + if not isinstance(value, str): + raise ValueError(f"{name} must be a UUID string.") + try: + identifier = uuid.UUID(value) + except ValueError: + raise ValueError(f"{name} must be a UUID string.") from None + if not identifier.int: + raise ValueError(f"{name} must be a non-nil UUID.") + return str(identifier) + + +def _integer(value, default, minimum, maximum, name): + if value is None or value == "": + return default + if isinstance(value, bool) or not re.fullmatch(r"[0-9]{1,3}", str(value)): + raise ValueError(f"{name} must be an integer.") + number = int(value) + if not minimum <= number <= maximum: + raise ValueError(f"{name} is outside its supported range.") + return number + + +def _settings(params): + if not isinstance(params, dict): + raise ValueError("params must be an object.") + source = params.get("source") + if source is None or source == "": + source = "misp-modules" + if not isinstance(source, str) or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9 ._:/-]{0,63}", source): + raise ValueError("source must be 1-64 ASCII letters, digits, spaces or ._:/- and start with a letter or digit.") + if source.lower().strip() == "microsoft sentinel": + raise ValueError("The source-system name Microsoft Sentinel is reserved by the API.") + return { + "workspace_id": _uuid(params.get("workspace_id"), "workspace_id"), + "source": source, + "days": _integer(params.get("expiration_days"), 30, 1, 365, "expiration_days"), + "confidence": _integer(params.get("confidence"), 50, 0, 100, "confidence"), + "dry_run": _boolean(params.get("dry_run")), + } + + +def _records(container, key): + records = container.get(key, []) + if not isinstance(records, list) or any(not isinstance(record, dict) for record in records): + raise ValueError(f"{key} must be a list of objects.") + return records + + +def _tags(*containers): + names = set() + for container in containers: + for key in ("Tag", "EventTag", "AttributeTag"): + for tag in _records(container, key): + tag = tag.get("Tag", tag) + if isinstance(tag, dict) and isinstance(tag.get("name"), str): + tag["name"].encode("utf-8") + if tag["name"]: + names.add(tag["name"]) + return names + + +def _domain(value): + domain = value.removesuffix(".").encode("idna").decode("ascii").lower() + if len(domain) > 253 or any( + not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label) for label in domain.split(".") + ): + raise ValueError("Invalid domain.") + return domain + + +def _atomic(kind, value): + value.encode("utf-8") + if any(ord(char) < 32 or ord(char) == 127 for char in value): + raise ValueError("Control characters are unsupported.") + if kind in HASH_LENGTHS: + algorithm, length = HASH_LENGTHS[kind] + if not re.fullmatch(rf"[0-9a-fA-F]{{{length}}}", value): + raise ValueError("Invalid hash.") + return f"file:hashes.'{algorithm}'", value.lower() + if kind in ("ip-src", "ip-dst"): + if "%" in value: + raise ValueError("Scoped IPv6 is unsupported.") + address = ipaddress.ip_address(value) + return f"ipv{address.version}-addr:value", str(address) + if kind in ("domain", "hostname"): + return "domain-name:value", _domain(value) + if kind in EMAIL_TYPES: + local, separator, domain = value.rpartition("@") + if not separator or not re.fullmatch(r"[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+", local): + raise ValueError("Unsupported email address.") + return "email-addr:value", local + "@" + _domain(domain) + parsed = urlsplit(value) + if parsed.scheme not in ("http", "https") or not parsed.hostname or any(char.isspace() for char in value): + raise ValueError("URL must be absolute HTTP(S) with a host and no whitespace.") + if parsed.username is not None or parsed.password is not None: + raise ValueError("Credential-bearing URLs are unsupported.") + parsed.port + return "url:value", value + + +def _atoms(kind, value): + if "|" not in kind and kind != "malware-sample": + return [_atomic(kind, value)] + first, separator, second = value.rpartition("|") + if not separator or not first.strip() or not second.strip(): + raise ValueError("Invalid composite.") + if kind == "domain|ip": + return [_atomic("domain", first), _atomic("ip-dst", second)] + return [_atomic("md5" if kind == "malware-sample" else kind.split("|")[1], second)] + + +def _stamp(value): + return value.isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _collect(data, settings, now): + if isinstance(data, dict): + data = [data] + if not isinstance(data, list) or any(not isinstance(envelope, dict) for envelope in data): + raise ValueError("data must contain an event or list of events.") + indicators, skipped = {}, Counter() + for envelope in data: + event = envelope.get("Event", envelope) + if not isinstance(event, dict): + raise ValueError("Event must be an object.") + if _boolean(event.get("deleted")): + continue + collections = dict(event, **envelope) + attributes = [(attribute, {}) for attribute in _records(collections, "Attribute")] + for obj in _records(collections, "Object"): + if not _boolean(obj.get("deleted")): + attributes.extend((attribute, obj) for attribute in _records(obj, "Attribute")) + for attribute, obj in attributes: + if _boolean(attribute.get("deleted")) or not _boolean(attribute.get("to_ids")): + skipped["deleted or non-IDS"] += 1 + continue + kind, value = attribute.get("type"), attribute.get("value") + if not isinstance(kind, str) or kind not in SUPPORTED_TYPES: + skipped["unsupported"] += 1 + continue + try: + identifier = _uuid(attribute.get("uuid"), "attribute UUID") + timestamp = attribute.get("timestamp") + if isinstance(timestamp, bool) or not re.fullmatch(r"[0-9]{1,10}", str(timestamp)): + raise ValueError("Attribute timestamp must be Unix seconds.") + created = datetime.fromtimestamp(int(timestamp), timezone.utc) + if created > now: + raise ValueError("Attribute timestamp is in the future.") + if not isinstance(value, str) or not value.strip(): + raise ValueError("Empty value.") + atoms = _atoms(kind, value) + except (ValueError, OverflowError, OSError): + skipped["malformed value or identity/timestamp"] += 1 + continue + labels = _tags(event, envelope, obj, attribute) + ref = { + "event": str(event.get("uuid") or event.get("id") or "unknown"), + "attribute": identifier, + "misp_type": kind, + } + if obj: + ref["object"] = str(obj.get("uuid") or obj.get("id") or obj.get("name") or "unknown") + for path, normalized in atoms: + escaped = normalized.replace("\\", "\\\\").replace("'", "\\'") + pattern = f"[{path} = '{escaped}']" + stix_id = "indicator--" + str( + uuid.uuid5( + uuid.NAMESPACE_URL, + json.dumps( + [ + "misp-sentinel-publish", + settings["workspace_id"], + settings["source"], + identifier, + str(int(timestamp)), + pattern, + ] + ), + ) + ) + entry = indicators.setdefault( + stix_id, {"pattern": pattern, "created": _stamp(created), "sources": [], "labels": set()} + ) + if ref not in entry["sources"]: + entry["sources"].append(ref) + entry["labels"].update(labels) + if len(indicators) > MAX_INDICATORS: + raise ValueError(f"At most {MAX_INDICATORS} unique indicators are allowed; split the input.") + objects, outcomes = [], [] + for identifier, entry in sorted(indicators.items()): + refs = sorted(entry["sources"], key=lambda ref: json.dumps(ref, sort_keys=True)) + item = { + "type": "indicator", + "spec_version": "2.1", + "id": identifier, + "created": entry["created"], + "modified": _stamp(now), + "pattern_type": "stix", + "pattern_version": "2.1", + "pattern": entry["pattern"], + "valid_from": entry["created"], + "valid_until": _stamp(now + timedelta(days=settings["days"])), + "confidence": settings["confidence"], + "name": "MISP " + identifier, + "description": json.dumps(refs, sort_keys=True), + "external_references": [{"source_name": "MISP", "external_id": ref["attribute"]} for ref in refs], + } + if entry["labels"]: + item["labels"] = sorted(entry["labels"]) + markings = {TLP_MARKINGS[label.lower()] for label in entry["labels"] if label.lower() in TLP_MARKINGS} + if markings: + item["object_marking_refs"] = sorted(markings) + objects.append(item) + outcomes.append({"id": identifier, "sources": refs}) + return objects, outcomes, dict(skipped) + + +def _request(session, endpoint, **kwargs): + try: + response = session.post(endpoint, timeout=15, verify=True, allow_redirects=False, **kwargs) + except (requests.RequestException, OSError): + raise PublishError("Request failed; remote outcome may be unknown. Check Sentinel before retrying.") from None + if response.status_code != 200: + raise PublishError(f"HTTP {response.status_code}; remote outcome may be unknown.") + return response + + +def _authenticate(session): + try: + tenant = _uuid(os.getenv("SENTINEL_TENANT_ID"), "tenant ID") + client = _uuid(os.getenv("SENTINEL_CLIENT_ID"), "client ID") + secret = os.getenv("SENTINEL_CLIENT_SECRET", "") + if not secret or any(ord(char) < 32 or ord(char) == 127 for char in secret): + raise ValueError("Invalid client secret.") + except ValueError: + raise PublishError( + "Configure SENTINEL_TENANT_ID, SENTINEL_CLIENT_ID and SENTINEL_CLIENT_SECRET on the module server." + ) from None + response = _request( + session, + f"{AUTH_ORIGIN}/{tenant}/oauth2/v2.0/token", + data={ + "grant_type": "client_credentials", + "client_id": client, + "client_secret": secret, + "scope": SCOPE, + }, + ) + try: + body = response.json() + token = body["access_token"] + if ( + body.get("token_type", "").lower() != "bearer" + or not isinstance(token, str) + or not token + or any(not 33 <= ord(char) <= 126 for char in token) + or type(body.get("expires_in")) is not int + or body["expires_in"] <= DEADLINE_SECONDS + 15 + ): + raise ValueError("Invalid token response.") + except (ValueError, KeyError, TypeError, AttributeError): + raise PublishError("Invalid or short-lived Entra token response; no indicators submitted.") from None + session.headers.update({"Authorization": f"Bearer {token}"}) + + +def _upload(session, endpoint, payload): + response = _request(session, endpoint, params={"api-version": API_VERSION}, json=payload) + if not response.content.strip(): + return set() + try: + body = response.json() + if not isinstance(body, dict) or set(body) != {"errors"} or not isinstance(body["errors"], list): + raise ValueError("Invalid response.") + failures = set() + for error in body["errors"]: + index = error["recordIndex"] + messages = error["errorMessages"] + if ( + type(index) is not int + or not 0 <= index < len(payload["stixobjects"]) + or index in failures + or not isinstance(messages, list) + or not messages + or any(not isinstance(message, str) for message in messages) + ): + raise ValueError("Invalid per-object result.") + failures.add(index) + return failures + except (ValueError, TypeError, KeyError): + raise PublishError( + "Invalid upload acknowledgement; remote outcome may be unknown. Check Sentinel before retrying." + ) from None + + +def handler(q=False): + if not q: + return False + try: + request = json.loads(q) + if not isinstance(request, dict): + raise ValueError("Request must be an object.") + if "filteredItems" in request or "matchingData" in request: + raise ValueError("Workflow filters/extracted data are unsupported; supply selected events in data.") + settings = _settings(request.get("params", {})) + now = datetime.now(timezone.utc) + objects, outcomes, skipped = _collect(request.get("data"), settings, now) + batches = [ + {"sourcesystem": settings["source"], "stixobjects": objects[offset : offset + BATCH_SIZE]} + for offset in range(0, len(objects), BATCH_SIZE) + ] + if any(len(json.dumps(batch).encode("utf-8")) > MAX_BATCH_BYTES for batch in batches): + raise ValueError("A batch exceeds the local 1 MiB limit; split the input or reduce metadata.") + except (ValueError, TypeError) as exc: + return {"error": f"Sentinel publish: {exc}"} + summary = { + "success": True, + "dry_run": settings["dry_run"], + "workspace_id": settings["workspace_id"], + "uploaded": 0, + "failed": 0, + "planned": 0, + "skipped": skipped, + "items": outcomes, + } + result = {"result_key": "sentinel_publish", "data": summary} + if settings["dry_run"]: + summary["planned"] = len(objects) + summary["payloads"] = batches + for item in outcomes: + item["status"] = "planned" + return result + if not objects: + return result + endpoint = f"{UPLOAD_ORIGIN}/workspaces/{settings['workspace_id']}/threat-intelligence-stix-objects:upload" + deadline = time.monotonic() + DEADLINE_SECONDS + with requests.Session() as session: + session.headers.update({"Accept": "application/json"}) + try: + _authenticate(session) + except PublishError as exc: + summary["success"] = False + result["error"] = f"Sentinel authentication: {exc}" + for item in outcomes: + item["status"] = "not_attempted" + return result + stopped = None + for batch_index, batch in enumerate(batches): + batch_items = outcomes[batch_index * BATCH_SIZE : (batch_index + 1) * BATCH_SIZE] + try: + if stopped is not None: + raise PublishError(stopped) + if time.monotonic() >= deadline: + raise PublishError("Publication time budget exceeded; batch not attempted.") + failures = _upload(session, endpoint, batch) + for index, item in enumerate(batch_items): + if index in failures: + item.update(status="failed", error="Sentinel rejected this indicator.") + summary["failed"] += 1 + else: + item["status"] = "uploaded" + summary["uploaded"] += 1 + except PublishError as exc: + for item in batch_items: + item.update(status="failed", error=str(exc)) + summary["failed"] += 1 + stopped = "Publication stopped after an uncertain or unsuccessful batch; batch not attempted." + if summary["failed"]: + summary["success"] = False + result["error"] = ( + f"Sentinel publish: {summary['failed']} failed; {summary['uploaded']} uploaded. Publication may be partial." + ) + return result + + +def introspection(): + return {"config": moduleconfig} + + +def version(): + return dict(moduleinfo, config=moduleconfig) diff --git a/tests/test_sentinel_publish.py b/tests/test_sentinel_publish.py new file mode 100644 index 000000000..e10186eee --- /dev/null +++ b/tests/test_sentinel_publish.py @@ -0,0 +1,552 @@ +import copy +import json +import subprocess +import sys +import uuid +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock +from urllib.parse import parse_qs + +import pytest +import requests +from pymisp import MISPObject +from tornado.testing import AsyncHTTPTestCase +from tornado.web import Application + +from misp_modules import __main__ as service +from misp_modules.modules.action_mod import export_sentinel as wrapper +from misp_modules.modules.action_mod import sentinel_publish as publish + +WORKSPACE = "22549473-0938-4938-8a04-225494730938" +TENANT = "c6a7d64d-8d28-4c7d-ae30-8efcbb28804d" +CLIENT = "fdb91a8a-e7bd-4c6e-b265-b286c3321879" +EVENT = "f1ef0409-c80a-4012-b401-b7d1050f4ed4" +ATTRIBUTE = "aa09819b-f2d8-4661-83ea-d4897cafebc8" + + +def attribute(kind="domain", value="example.org", **metadata): + return {"uuid": ATTRIBUTE, "timestamp": "1767225600", "type": kind, "value": value, "to_ids": True, **metadata} + + +def event(attributes=None, objects=None, **metadata): + return {"uuid": EVENT, "Attribute": attributes or [], "Object": objects or [], **metadata} + + +def request(data=None, **params): + return json.dumps( + {"data": event([attribute()]) if data is None else data, "params": {"workspace_id": WORKSPACE, **params}} + ) + + +def run(data=None, **params): + return publish.handler(request(data, **params)) + + +def response(body=None, status=200, raw=None): + result = requests.Response() + result.status_code = status + result.headers["Content-Type"] = "application/json" + result._content = (b"" if body is None else json.dumps(body).encode()) if raw is None else raw + return result + + +@pytest.fixture +def client(monkeypatch): + session = MagicMock(spec=requests.Session) + session.headers = {} + session.__enter__.return_value = session + + def post(url, **kwargs): + if "oauth2" in url: + return response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}) + return response() + + session.post.side_effect = post + session.constructor = MagicMock(return_value=session) + monkeypatch.setattr(publish.requests, "Session", session.constructor) + monkeypatch.setenv("SENTINEL_TENANT_ID", TENANT) + monkeypatch.setenv("SENTINEL_CLIENT_ID", CLIENT) + monkeypatch.setenv("SENTINEL_CLIENT_SECRET", "client-secret") + return session + + +def payloads(result): + assert "error" not in result, result + return [obj for batch in result["data"]["payloads"] for obj in batch["stixobjects"]] + + +def test_direct_contract_uses_correct_authority_scope_endpoint_and_empty_body_ack(client): + result = run() + assert result["data"]["uploaded"] == 1 and result["data"]["success"] is True + assert result["result_key"] == "sentinel_publish" + auth, upload = client.post.call_args_list + assert auth.args == (f"https://login.microsoftonline.com/{TENANT}/oauth2/v2.0/token",) + assert auth.kwargs["data"] == { + "grant_type": "client_credentials", + "client_id": CLIENT, + "client_secret": "client-secret", + "scope": "https://management.azure.com/.default", + } + assert upload.args == ( + f"https://api.ti.sentinel.azure.com/workspaces/{WORKSPACE}/threat-intelligence-stix-objects:upload", + ) + assert upload.kwargs["params"] == {"api-version": "2024-02-01-preview"} + assert set(upload.kwargs["json"]) == {"sourcesystem", "stixobjects"} + assert upload.kwargs["json"]["sourcesystem"] == "misp-modules" + obj = upload.kwargs["json"]["stixobjects"][0] + expiry = datetime.fromisoformat(obj["valid_until"].replace("Z", "+00:00")) + assert abs((expiry - datetime.now(timezone.utc) - timedelta(days=30)).total_seconds()) < 5 + assert client.headers["Authorization"] == "Bearer access-token" + assert all( + call.kwargs["verify"] is True and call.kwargs["allow_redirects"] is False and call.kwargs["timeout"] == 15 + for call in [auth, upload] + ) + client.__exit__.assert_called_once() + + +@pytest.mark.parametrize( + "kind,value,pattern", + [ + ("domain", "EXAMPLE.org.", "[domain-name:value = 'example.org']"), + ("hostname", "bücher.example", "[domain-name:value = 'xn--bcher-kva.example']"), + ("ip-src", "192.0.2.1", "[ipv4-addr:value = '192.0.2.1']"), + ("ip-dst", "2001:0DB8:0000::1", "[ipv6-addr:value = '2001:db8::1']"), + ("url", "https://example.org/A?q=1", "[url:value = 'https://example.org/A?q=1']"), + ("email-src", "User@EXAMPLE.org", "[email-addr:value = 'User@example.org']"), + ("url", "https://example.org/a'b", "[url:value = 'https://example.org/a\\'b']"), + ("url", "https://example.org/a\\b", "[url:value = 'https://example.org/a\\\\b']"), + *[ + (kind, "A" * length, f"[file:hashes.'{algorithm}' = '" + "a" * length + "']") + for kind, (algorithm, length) in publish.HASH_LENGTHS.items() + ], + *[ + (f"filename|{kind}", "a|b.exe|" + "a" * length, f"[file:hashes.'{algorithm}' = '" + "a" * length + "']") + for kind, (algorithm, length) in publish.HASH_LENGTHS.items() + ], + ("malware-sample", "file.exe|" + "a" * 32, "[file:hashes.'MD5' = '" + "a" * 32 + "']"), + ], +) +def test_supported_values_generate_valid_stix_patterns(kind, value, pattern): + stix2 = pytest.importorskip("stix2") + obj = payloads(run(event([attribute(kind, value)]), dry_run=True))[0] + assert obj["pattern"] == pattern + parsed = stix2.parse(obj) + assert parsed.type == "indicator" and parsed.spec_version == "2.1" + + +def test_domain_ip_and_object_provenance_are_preserved_without_mutation(): + obj = {"uuid": "object-1", "name": "domain-ip", "Attribute": [attribute("domain|ip", "example.org|192.0.2.1")]} + data = [{"Event": event(objects=[obj])}, {"Event": {"uuid": EVENT}, "Object": [obj]}, event(objects=[obj])] + original = copy.deepcopy(data) + result = run(data, dry_run=True) + objects = payloads(result) + assert len(objects) == 2 + assert all( + len(item["sources"]) == 1 and item["sources"][0]["object"] == "object-1" for item in result["data"]["items"] + ) + assert data == original + + +@pytest.mark.parametrize( + "template,relation,value", [("file", "sha256", "a" * 64), ("domain-ip", "domain", "example.org")] +) +def test_existing_misp_object_templates_suffice(template, relation, value): + obj = MISPObject(template, strict=True) + obj.add_attribute(relation, value=value, to_ids=True, uuid=ATTRIBUTE, timestamp=1767225600) + obj._validate() + assert len(payloads(run(event(objects=[json.loads(obj.to_json())]), dry_run=True))) == 1 + + +def test_source_revision_ids_and_created_are_stable_but_change_with_content_timestamp_or_namespace(): + first = payloads(run(dry_run=True))[0] + again = payloads(run(dry_run=True, expiration_days=45, confidence=75))[0] + assert first["id"] == again["id"] and first["created"] == again["created"] == "2026-01-01T00:00:00.000Z" + assert first["modified"] <= again["modified"] + assert again["confidence"] == 75 and again["valid_until"] > first["valid_until"] + for data, params in [ + (event([attribute(value="other.example")]), {}), + (event([attribute(timestamp="1767225601")]), {}), + (event([attribute(uuid=str(uuid.uuid4()))]), {}), + (event([attribute()]), {"source": "another publisher"}), + (event([attribute()]), {"workspace_id": str(uuid.uuid4())}), + ]: + assert payloads(run(data, dry_run=True, **params))[0]["id"] != first["id"] + + +def test_tags_and_supported_tlp_markings_are_preserved_without_private_comments_or_samples(): + data = event( + [attribute(Tag=[{"name": "tlp:green"}], comment="private-comment", data="private-sample")], + Tag=[{"name": "event-tag"}], + EventTag=[{"Tag": {"name": "tlp:amber"}}], + info="private-title", + ) + obj = payloads(run(data, dry_run=True))[0] + assert obj["labels"] == ["event-tag", "tlp:amber", "tlp:green"] + assert set(obj["object_marking_refs"]) == {publish.TLP_MARKINGS["tlp:green"], publish.TLP_MARKINGS["tlp:amber"]} + assert not any(word in json.dumps(obj) for word in ["private-comment", "private-sample", "private-title"]) + + +def test_dry_run_needs_no_credentials_or_requests_session(client, monkeypatch): + for name in ["SENTINEL_TENANT_ID", "SENTINEL_CLIENT_ID", "SENTINEL_CLIENT_SECRET"]: + monkeypatch.delenv(name) + assert run(dry_run=True)["data"]["planned"] == 1 + client.constructor.assert_not_called() + + +def test_batches_are_limited_to_one_hundred_and_share_one_authentication(client): + data = event( + [ + attribute(value=f"host{index}.example", uuid=str(uuid.uuid5(uuid.NAMESPACE_URL, str(index)))) + for index in range(205) + ] + ) + result = run(data) + assert result["data"]["uploaded"] == 205 + assert [len(call.kwargs["json"]["stixobjects"]) for call in client.post.call_args_list[1:]] == [100, 100, 5] + assert client.post.call_count == 4 + + +@pytest.mark.parametrize( + "body", + [ + {"errors": [{"recordIndex": 0, "errorMessages": ["client-secret"]}]}, + {"errors": [{"recordIndex": 1, "errorMessages": ["access-token"]}]}, + ], +) +def test_http_200_partial_failures_are_mapped_by_record_index_without_exposing_messages(client, body): + client.post.side_effect = [ + response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}), + response(body), + ] + result = run(event([attribute(), attribute(value="other.example", uuid=str(uuid.uuid4()))])) + assert result["data"]["uploaded"] == 1 and result["data"]["failed"] == 1 + assert result["data"]["success"] is False and "partial" in result["error"] + assert result["data"]["items"][body["errors"][0]["recordIndex"]]["status"] == "failed" + assert "client-secret" not in json.dumps(result) and "access-token" not in json.dumps(result) + + +@pytest.mark.parametrize( + "body", + [ + {}, + [], + {"unexpected": True}, + {"errors": {}}, + {"errors": [{}]}, + {"errors": [{"recordIndex": True, "errorMessages": ["bad"]}]}, + {"errors": [{"recordIndex": -1, "errorMessages": ["bad"]}]}, + {"errors": [{"recordIndex": 1, "errorMessages": ["bad"]}]}, + {"errors": [{"recordIndex": 0, "errorMessages": []}]}, + {"errors": [{"recordIndex": 0, "errorMessages": "bad"}]}, + {"errors": [{"recordIndex": 0, "errorMessages": ["bad"]}] * 2}, + {"errors": [], "warnings": ["bad"]}, + ], +) +def test_malformed_upload_acknowledgements_are_uncertain_failures(client, body): + client.post.side_effect = [ + response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}), + response(body), + ] + result = run() + assert result["data"]["failed"] == 1 and result["data"]["uploaded"] == 0 + assert "unknown" in result["data"]["items"][0]["error"] + assert client.post.call_count == 2 + + +def test_empty_validation_error_list_is_an_acknowledgement(client): + client.post.side_effect = [ + response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}), + response({"errors": []}), + ] + assert run()["data"]["uploaded"] == 1 + + +@pytest.mark.parametrize("status", [202, 204, 301, 302, 400, 401, 403, 404, 429, 500]) +def test_upload_http_failure_stops_later_batches_without_retry_or_echoing_secrets(client, status): + client.post.side_effect = [ + response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}), + response({"message": "client-secret"}, status), + ] + result = run(event([attribute(value=f"host{index}.example") for index in range(101)])) + assert result["data"]["failed"] == 101 and result["data"]["uploaded"] == 0 + assert "not attempted" in result["data"]["items"][-1]["error"] + assert "client-secret" not in json.dumps(result) and client.post.call_count == 2 + + +@pytest.mark.parametrize( + "failure", + [ + requests.Timeout("client-secret"), + requests.ConnectionError("access-token"), + OSError("client-secret"), + response(raw=b"not JSON"), + ], +) +def test_unknown_write_outcomes_preserve_prior_success(client, failure): + client.post.side_effect = [ + response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}), + response(), + failure, + ] + result = run(event([attribute(value=f"host{index}.example") for index in range(101)])) + assert result["data"]["uploaded"] == 100 and result["data"]["failed"] == 1 + assert "client-secret" not in json.dumps(result) and "access-token" not in json.dumps(result) + + +@pytest.mark.parametrize( + "body", + [ + {}, + [], + {"access_token": "access-token"}, + {"access_token": "access-token", "token_type": "other", "expires_in": 3600}, + {"access_token": "access-token\r\nHeader: bad", "token_type": "Bearer", "expires_in": 3600}, + {"access_token": "access-token", "token_type": "Bearer", "expires_in": 10}, + {"access_token": "access-token", "token_type": "Bearer", "expires_in": True}, + ], +) +def test_invalid_token_responses_do_not_upload_or_expose_secrets(client, body): + client.post.side_effect = [response(body)] + result = run() + assert "error" in result and result["data"]["success"] is False + assert result["data"]["items"][0]["status"] == "not_attempted" + assert "access-token" not in json.dumps(result) and client.post.call_count == 1 + + +@pytest.mark.parametrize( + "key,value", + [ + ("SENTINEL_TENANT_ID", "bad"), + ("SENTINEL_CLIENT_ID", "bad"), + ("SENTINEL_CLIENT_SECRET", ""), + ("SENTINEL_CLIENT_SECRET", "secret\r\n"), + ], +) +def test_invalid_server_credentials_never_reach_network(client, monkeypatch, key, value): + monkeypatch.setenv(key, value) + assert "error" in run() + client.post.assert_not_called() + + +@pytest.mark.parametrize( + "params", + [ + {"workspace_id": "bad/path"}, + {"workspace_id": "00000000-0000-0000-0000-000000000000"}, + {"source": "Microsoft Sentinel"}, + {"source": []}, + {"source": "a" * 65}, + {"expiration_days": 0}, + {"expiration_days": 366}, + {"confidence": -1}, + {"confidence": 101}, + {"confidence": True}, + {"dry_run": "maybe"}, + ], +) +def test_invalid_direct_configuration_precedes_network(client, params): + assert "error" in run(**params) + client.constructor.assert_not_called() + + +@pytest.mark.parametrize( + "metadata", + [ + {"uuid": None}, + {"uuid": "bad"}, + {"timestamp": None}, + {"timestamp": True}, + {"timestamp": "9999999999"}, + {"value": "bad..example"}, + {"value": "\ud800"}, + ], +) +def test_bad_source_identity_timestamp_or_value_is_skipped(client, metadata): + result = run(event([attribute(**metadata)])) + assert result["data"]["skipped"] == {"malformed value or identity/timestamp": 1} + client.constructor.assert_not_called() + + +def test_deleted_non_ids_and_unsupported_constraints_are_excluded(client): + data = [ + event( + [attribute(deleted=True), attribute(to_ids="false"), attribute("ip-dst|port", "192.0.2.1|443")], + [{"deleted": True, "Attribute": [attribute()]}], + ), + event([attribute()], deleted=True), + ] + assert run(data)["data"]["skipped"] == {"deleted or non-IDS": 2, "unsupported": 1} + client.constructor.assert_not_called() + + +def test_limits_and_malformed_containers_are_checked_before_network(client, monkeypatch): + assert "error" in run(event([attribute(value=f"host{index}.example") for index in range(1001)])) + monkeypatch.setattr(publish, "MAX_BATCH_BYTES", 10) + assert "error" in run() + for payload in [None, "bad", [1], {"Event": "bad"}, {"Object": [1]}]: + assert "error" in publish.handler(json.dumps({"data": payload, "params": {"workspace_id": WORKSPACE}})) + client.constructor.assert_not_called() + + +def test_deadline_is_checked_after_authentication(client, monkeypatch): + values = iter([0, publish.DEADLINE_SECONDS + 1]) + monkeypatch.setattr(publish.time, "monotonic", lambda: next(values)) + assert run()["data"]["failed"] == 1 + assert client.post.call_count == 1 + + +@pytest.mark.parametrize("module", [publish, wrapper]) +def test_request_validation_and_unapplied_filters(module): + assert module.handler() is False + for payload in [ + "invalid JSON", + "[]", + "null", + json.dumps({"params": []}), + json.dumps({"filteredItems": []}), + json.dumps({"matchingData": []}), + ]: + assert "error" in module.handler(payload) + + +def test_wrapper_direct_mode_uses_same_publisher_and_does_not_spawn(client, monkeypatch): + process = MagicMock() + monkeypatch.setattr(wrapper.subprocess, "run", process) + result = wrapper.handler(request(mode="direct")) + assert result["data"]["uploaded"] == 1 and result["result_key"] == "sentinel_publish" + process.assert_not_called() + + +@pytest.mark.parametrize( + "params,data", + [ + ({"mode": "unknown"}, {"Event": {"uuid": EVENT}}), + ({"dry_run": True}, {"Event": {"uuid": EVENT}}), + ({"misp2_venv": "relative"}, {"Event": {"uuid": EVENT}}), + ({"misp2_script": "/bad\x00path"}, {"Event": {"uuid": EVENT}}), + ({}, {"Event": {"uuid": "bad"}}), + ({}, [{"Event": {"uuid": EVENT}}]), + ], +) +def test_wrapper_invalid_mode_paths_or_input_never_execute(monkeypatch, params, data): + process = MagicMock() + monkeypatch.setattr(wrapper.subprocess, "run", process) + assert "error" in wrapper.handler(json.dumps({"data": data, "params": params})) + process.assert_not_called() + + +def test_missing_wrapper_installation_returns_a_workflow_error(monkeypatch): + monkeypatch.setattr(wrapper.os.path, "isfile", lambda path: False) + assert wrapper.handler(json.dumps({"data": {"Event": {"uuid": EVENT}}}))["data"] is False + + +def test_legacy_wrapper_default_calls_python_with_event_uuid_and_boolean_success(monkeypatch): + monkeypatch.setattr(wrapper.os.path, "isfile", lambda path: True) + process = MagicMock(return_value=MagicMock(returncode=0)) + monkeypatch.setattr(wrapper.subprocess, "run", process) + result = wrapper.handler( + json.dumps( + { + "data": {"Event": {"uuid": EVENT}}, + "params": {"misp2_venv": "/opt/env", "misp2_script": "/opt/app/script.py"}, + } + ) + ) + assert result == {"data": True} + assert process.call_args.args[0] == ["/opt/env/bin/python", "/opt/app/script.py", EVENT] + assert process.call_args.kwargs["cwd"] == "/opt/app" + assert process.call_args.kwargs["stdout"] == process.call_args.kwargs["stderr"] == subprocess.DEVNULL + + +@pytest.mark.parametrize( + "outcome", [MagicMock(returncode=1), subprocess.TimeoutExpired("script", 240), OSError("client-secret")] +) +def test_wrapper_failures_signal_workflow_error_without_logging_provider_output(monkeypatch, capsys, outcome): + monkeypatch.setattr(wrapper.os.path, "isfile", lambda path: True) + process = MagicMock() + if isinstance(outcome, Exception): + process.side_effect = outcome + else: + process.return_value = outcome + monkeypatch.setattr(wrapper.subprocess, "run", process) + result = wrapper.handler(json.dumps({"data": {"Event": {"uuid": EVENT}}, "params": {}})) + assert result["data"] is False and "error" in result and "client-secret" not in json.dumps(result) + assert capsys.readouterr().out == "" + + +def test_real_wrapper_process_remains_compatible(tmp_path): + venv = tmp_path / "venv" + (venv / "bin").mkdir(parents=True) + (venv / "bin" / "python").symlink_to(sys.executable) + script = tmp_path / "script.py" + script.write_text("import sys\nfrom pathlib import Path\nPath('received.txt').write_text(sys.argv[1])\n") + result = wrapper.handler( + json.dumps( + {"data": {"Event": {"uuid": EVENT}}, "params": {"misp2_venv": str(venv), "misp2_script": str(script)}} + ) + ) + assert result == {"data": True} and (tmp_path / "received.txt").read_text() == EVENT + + +def test_actual_requests_wire_contract_has_form_auth_and_json_upload(monkeypatch): + captured = [] + + def send(self, req, **kwargs): + captured.append(req) + if "oauth2" in req.url: + assert "Authorization" not in req.headers + assert parse_qs(req.body)["client_secret"] == ["client-secret"] + result = response({"access_token": "access-token", "token_type": "Bearer", "expires_in": 3600}) + else: + assert ( + req.headers["Authorization"] == "Bearer access-token" + and req.headers["Content-Type"] == "application/json" + ) + assert "api-version=2024-02-01-preview" in req.url + assert len(json.loads(req.body)["stixobjects"]) == 1 + result = response() + result.request, result.url = req, req.url + return result + + for name, value in [ + ("SENTINEL_TENANT_ID", TENANT), + ("SENTINEL_CLIENT_ID", CLIENT), + ("SENTINEL_CLIENT_SECRET", "client-secret"), + ]: + monkeypatch.setenv(name, value) + monkeypatch.setattr(requests.adapters.HTTPAdapter, "send", send) + assert run()["data"]["uploaded"] == 1 and len(captured) == 2 + + +class TestSentinelPublishHTTP(AsyncHTTPTestCase): + def get_app(self): + self.previous, self.cache = service.MODULES_HANDLERS.copy(), service.ListModules.CACHE + service.MODULES_HANDLERS.clear() + service.MODULES_HANDLERS.update( + { + "sentinel_publish": publish, + "type:sentinel_publish": "action_mod", + "export_sentinel": wrapper, + "type:export_sentinel": "action_mod", + } + ) + service.ListModules.CACHE = None + return Application([(r"/modules", service.ListModules), (r"/query", service.QueryModule)]) + + def tearDown(self): + service.MODULES_HANDLERS.clear() + service.MODULES_HANDLERS.update(self.previous) + service.ListModules.CACHE = self.cache + super().tearDown() + + def test_discovery_and_direct_mode_query(self): + descriptors = {item["name"]: item for item in json.loads(self.fetch("/modules").body)} + assert set(descriptors) == {"sentinel_publish", "export_sentinel"} + assert descriptors["sentinel_publish"]["meta"]["config"]["blocking"] is True + for name in descriptors: + payload = json.loads(request(mode="direct", dry_run=True)) + payload["module"] = name + result = self.fetch( + "/query", method="POST", headers={"Content-Type": "application/json"}, body=json.dumps(payload) + ) + assert result.code == 200 and json.loads(result.body)["data"]["planned"] == 1