OCA框架系列文章之Kestrel威胁狩猎语言 详细介绍和分析（四）

# create four process entities in Kestrel and store them in the variable `proclist`proclist = NEW process [ {"name": "cmd.exe", "pid": "123"}                       , {"name": "explorer.exe", "pid": "99"}                       , {"name": "firefox.exe", "pid": "201"}                       , {"name": "chrome.exe", "pid": "205"}                       ]# match a pattern of browser processes, and put the matched entities in variable `browsers`browsers = GET process FROM proclist WHERE [process:name IN ('firefox.exe', 'chrome.exe')]# display the information (attributes name, pid) of the entities in variable `browsers`DISP browsers ATTR name, pid

$ kestrel helloworld.hf

name pidchrome.exe 205firefox.exe 201[SUMMARY] block executed in 1 secondsVARIABLE    TYPE  #(ENTITIES)  #(RECORDS)  process*proclist process            4           4         0browsers process            2           2         0*Number of related records cached.

name = "powershell.exe"

ps = GET process     FROM stixshifter://edp1     WHERE name = "powershell.exe"     START 2022-11-11T15:05:00Z STOP 2022-11-12T08:00:00Z

ps1 = GET process FROM stixshifter://edp1 WHERE name = "powershell.exe"      START 2022-11-11T15:05:00Z STOP 2022-11-12T08:00:00Zps2 = GET process  FROM stixshifter://edp1            WHERE name =  "powershell.exe"        START 2022-11-11T15:05:00Z    STOP 2022-11-12T08:00:00Z

name = 'powershell.exe'name = "powershell.exe"

ps3 = GET process      FROM stixshifter://edp1      WHERE process:name = 'powershell.exe'      START 2022-11-11T15:05:00Z STOP 2022-11-12T08:00:00Z

ps4 = GET process      FROM stixshifter://edp1      WHERE [process:name = 'powershell.exe']      START 2022-11-11T15:05:00Z STOP 2022-11-12T08:00:00Z

# a single (process) node graph patternproc1 = GET process        FROM stixshifter://edp1        WHERE name = "powershell.exe" AND pid = 1234        START 2022-11-11T15:05:00Z STOP 2022-11-12T08:00:00Z# a single (network-traffic) node graph pattern# this pattern is equivalent to `dst_port IN (80, 443)`netflow1 = GET network-traffic           FROM stixshifter://gateway1           WHERE dst_port = 80 OR dst_port = 443           START 2022-11-11T15:05:00Z STOP 2022-11-12T08:00:00Z# a single (file) node graph patternminikatz = GET file           FROM stixshifter://edp1           WHERE name = "C:ProgramDatap.exe"              OR hashes.MD5 IN ( "1a4fe4413a92d478625d97b7df1bd0cf"                               , "b6ff8f31007a3629a3c4be8999001ec9"                               , "e8994399f1656e58f72443b8861ce5d1"                               , "9ae602fddb5d2f9b63c5eb6aad0a2612"                               )           START 2022-11-11T15:05:00Z STOP 2022-11-12T08:00:00Z# a single (user-account) node graph patternusers = GET user-account        FROM stixshifter://authlogs        WHERE (user_id = 1001 AND account_login = "Tracy")           OR  user_id = 0           OR (user_id = 1003 AND is_privileged = true)           OR (account_login = "JJ" AND is_privileged = true)        START 2022-11-11T15:05:00Z STOP 2022-11-12T08:00:00Z

procs = GET process FROM stixshifter://edp1        WHERE name = 'cmd.exe'                                     # blue attribute          AND binary_ref.name MATCHES '.+.(exe|dll|bat)$'         # orange branch          AND opened_connection_refs[*].dst_ref.value = '10.1.1.1' # green branch          AND ( ( parent_ref.name = 'explorer.exe' AND             # yellow branch                  parent_ref.binary_ref.name = 'explorer.exe'      # lightblue branch                ) OR                ( parent_ref.name LIKE '%.exe' AND                 # yellow branch                  parent_ref.binary_ref.name != 'powershell.exe'   # lightblue branch                )              )        START 2022-11-11T15:05:00Z STOP 2022-11-12T08:00:00Z

procs = GET process FROM stixshifter://edp1        WHERE name = 'cmd.exe'          AND parent_ref.name = 'explorer.exe'          AND parent_ref.parent_ref.name = 'abc.exe'

procs = GET process FROM stixshifter://edp1        WHERE name = 'cmd.exe'                                     # blue attribute          AND binary_ref.name MATCHES '.+.(exe|dll|bat)$'         # orange branch          AND opened_connection_refs[*].dst_ref.value = '10.1.1.1' # green branch          AND ipv4-addr:value NOT ISSUBSET '192.168.0.0/24'        # red subgraph          AND ( ( parent_ref.name = 'explorer.exe' AND             # yellow branch                  parent_ref.binary_ref.name = 'explorer.exe'      # lightblue branch                ) OR                ( parent_ref.name LIKE '%.exe' AND                 # yellow branch                  parent_ref.binary_ref.name != 'powershell.exe'   # lightblue branch                )              )          AND email-message:from_ref.value = '[email protected]'       # purple subgraph        START 2022-11-11T15:05:00Z STOP 2022-11-12T08:00:00Z

x-oca-asset:hostname = 'endpoint101'

# basic usage# `px` is a Kestrel variable of processespy1 = GET process FROM stixshifter://edp      WHERE pid = px.pid# both `=` and `IN` are valid to use as operator for referred variable# py2 returns the same as py1py2 = GET process FROM stixshifter://edp      WHERE pid IN px.pid# nested list is valid to use# all values will be flattened when parsedpy3 = GET process FROM stixshifter://edp      WHERE pid IN (123, px.pid, (4, 10548))

# hunting with data source `stixshifter://edp1`procs1 = ...nt1 = FIND network-traffic CREATED BY procs1# display the source/destination IP/port# this is for human inspection purposeDISP nt1 ATTR src_ref.value, src_port, dst_ref.value, dst_port# get the other end of the network traffic, not in edp1 data, but in edp2 data/view# use <src IP, src port, dst IP, dst port, time> to uniquely identify the traffic# time is automatically inferred by Kestrel due to variable referencent2 = GET network-traffic      FROM stixshifter://edp2      WHERE src_ref.value = nt1.src_ref.value        AND src_port      = nt1.src_port        AND dst_ref.value = nt1.dst_ref.value        AND dst_port      = nt1.dst_port# more generally, <src_port, time> is usually sufficient as the unique identifier# time is automatically inferred by Kestrel due to variable reference# `nt2x` usually gets the same results as `nt2`nt2x = GET network-traffic       FROM stixshifter://edp2       WHERE src_ref.value = nt1.src_ref.value# now get the processes handling the traffic on `edp2` and print their command lineprocs2 = FIND process CREATED nt2DISP procs2 ATTR command_line

# get the network traffic from `stixshifter://edp` to inspectnt_inner = ...# get the outter half of network traffic from the proxy using variable referencent_outter = GET network-traffic            FROM stixshifter://pxy            WHERE src_ref.value = nt_inner.src_ref.value              AND src_port = nt_inner.src_port# display the real remote IP for human inspectionDISP nt_outter ATTR dst_ref.value, dst_port# enrich the IPs in network-traffic with x-force threat intelligenceAPPLY python://xfeipenrich ON nt_outter

# the following will generate a STIX pattern# [process:command_line = 'powershell.exe "yes args"']pe1 = GET process FROM stixshifter://edp1      WHERE command_line = "powershell.exe "yes args""# an easier way is to use single quote for string literal# when there are double quotes in the string# pe2 is the same as pe1pe2 = GET process FROM stixshifter://edp1      WHERE command_line = 'powershell.exe "yes args"'# the following will generate a STIX pattern# [process:command_line = 'powershell.exe 'yes args'']pe3 = GET process FROM stixshifter://edp1      WHERE command_line = "powershell.exe 'yes args'"# backslash always needs to be escapedpe4 = GET process FROM stixshifter://edp1      WHERE command_line = "C:\Windows\System32\cmd.exe"# `.` is the dot character in regex# use `\.` since `` needs to be escapedps5 = GET process FROM stixshifter://edp1      WHERE name MATCHES 'cmd\.exe'# another regex escaping example that uses `w` and `.`ps5 = GET process FROM stixshifter://edp1      WHERE name MATCHES '\w+\.exe'

# f1 and f2 describes the same pattern:# using regex to match an exact string 'C:WindowsSystem32cmd.exe'f1 = GET file FROM stixshifter://edp1     WHERE name MATCHES 'C:\\Windows\\System32\\cmd\.exe'f2 = GET file FROM stixshifter://edp1     WHERE name MATCHES r'C:\Windows\System32\cmd.exe'# raw string can be used not only in regex (keyword MATCHES), but any comparison expression# f3/f4 will get the same results as f1/f2, yet they use exact match instead of regexf3 = GET file FROM stixshifter://edp1     WHERE name = 'C:\Windows\System32\cmd.exe'f4 = GET file FROM stixshifter://edp1     WHERE name = r'C:WindowsSystem32cmd.exe'

returned_variable = GET returned_entity_type [FROM entity_pool] WHERE ecgp [time_range] [LIMIT limit]

# get processes from host101 which has a parent process with name 'abc.exe'procs = GET process FROM stixshifter://host101 WHERE parent_ref.name = 'abc.exe'        START 2021-05-06T00:00:00Z STOP 2021-05-07T00:00:00Z# get files from a sealed STIX bundle with hash 'dbfcdd3a1ef5186a3e098332b499070a'# Kestrel allows to write a command in multiple linesbinx = GET file       FROM https://a.com/b.json       WHERE hashes.MD5 = 'dbfcdd3a1ef5186a3e098332b499070a'       START 2021-05-06T00:00:00Z STOP 2021-05-07T00:00:00Z# get processes from the above procs variable with pid 10578 and name 'xyz'# usually no time range is used when the entity pool is a varibleprocs2 = GET process FROM procs WHERE pid = 10578 AND name = 'xyz'# refer to another Kestrel variable in the WHERE clause (ECGP)# Kestrel will infer time range from `procs2`; users can override it by providing oneprocs3 = GET process FROM procs WHERE pid = procs2.pid# omitting the FROM clause, which will be desugarred as 'FROM https://a.com/b.json'procs4 = GET process WHERE pid = 1234         START 2021-05-06T00:00:00Z STOP 2021-05-07T00:00:00Z

returned_variable = FIND returned_entity_type RELATIONFROM input_variable [WHERE ecgp] [time_range] [LIMIT limit]

# find parent processes of processes in procsparent_procs = FIND process CREATED procs# find child processes of processes in procsparent_procs = FIND process CREATED BY procs# find network-traffic associated with processes in procsnt = FIND network-traffic CREATED BY procs# find processes associated with network-traffic in ntntprocs = FIND process CREATED network-traffic# find source IP addresses in ntsrc_ip = FIND ipv4-addr CREATED nt# find destination IP addresses in ntsrc_ip = FIND ipv4-addr ACCEPTED nt# find both source and destination IP addresses in ntsrc_ip = FIND ipv4-addr LINKED nt# find network-traffic which have source IP src_ipntspecial = FIND network-traffic CREATED BY src_ip

# find parent processes of processes in procs## the added WHERE clause limits the search to be performed against endpoint101## if there are other endpoints data in the data source (used to get `procs`),# they will not be matched against## assume the process identifier such as pid is reused across endpoints,# this will reduce false positives and avoid unnecessary computation/transmision#parent_procs_ww = FIND process CREATED procs                  WHERE x-oca-asset:hostname = 'endpoint101'

# the centered subgraph `process:name = 'bash'` in the following command# will be abandoned when executing, resulting parent_procs_ww2 == parent_procs_wwparent_procs_ww2 = FIND process CREATED procs                   WHERE name = 'bash' AND x-oca-asset:hostname = 'endpoint101'

parent_procs_ww = FIND process CREATED procs                  WHERE x-oca-asset:hostname = 'endpoint101'parent_procs_bash = parent_procs_ww WHERE name = 'bash'

# some early hunt stepsnt = GET network-traffic     FROM stixshifter://edp     WHERE dst_ref.value = '10.10.30.1'     LAST 5 DAY# it is OK to write this FIND without time range# which only search for the time range of `nt` for any records of `p1`p1 = FIND process CREATED nt# then, `pp1` will be empty (if the process is created 10 days ago)# - `p1` is assocaited with time range inferred from `nt` (last 5 days)# - no record in the last 5 days is about process creation of `p1`# - so Kestrel cannot grab anything about the parent process of `p1`pp1 = FIND process CREATED p1# alternatively, override the time range when retrieving data for `p2`# telling Kestrel to search for all `p2` records within the last 10 daysp2 = FIND process CREATED nt LAST 10 DAY# now the parent process will be discoveredpp2 = FIND process CREATED p2

returned_variable = NEW [returned_entity_type] data

# create a list of processes with their namesnewprocs = NEW process ["cmd.exe", "explorer.exe", "google-chrome.exe"]# create a list of processes with a list of dictionariesnewvar = NEW [ {"type": "process", "name": "cmd.exe", "pid": "123"}             , {"type": "process", "name": "explorer.exe", "pid": "99"}             ]# return entity type is required if not a key in the datanewvar2 = NEW process [ {"name": "abc.exe", "pid": "1234"}                      , {"name": "ie.exe", "pid": "10"}                      ]

newvar = SORT varx BY attribute [ASC|DESC]

# get network traffic and sort them by their destination portnt = GET network-traffic FROM stixshifter://idsX WHERE dst_ref_value = '1.2.3.4'ntx = SORT nt BY dst_port ASC# display all destination port and now it is easy to check important portsDISP ntx ATTR dst_port

aggr_var = GROUP varx BY attr1, attr2... [WITH aggr_fun(attr3) [AS alias], ...]aggr_var = GROUP varx BY BIN(attr, bin_size [time unit])... [WITH aggr_fun(attr3) [AS alias], ...]

# group processes by their name and displayprocs = GET process FROM stixshifter://edrA WHERE parent_ref.name = 'bash'aggr = GROUP procs BY nameDISP aggr ATTR unique_name, unique_pid, unique_command_line# group network traffic into 5 minute buckets:conns = GET network-traffic FROM stixshifter://my_ndr WHERE src_ref.value LIKE '%'conns_ts = TIMESTAMPED(conns)conns_binned = GROUP conns_ts BY BIN(first_observed, 5m) WITH COUNT(src_port) AS count

APPLY analytics_identifier ON var1, var2, ... WITH x=abc, y=[1,2,3], z=varx.pid

INFO varx

# showing information like attributes and how many entities in a variablent = GET network-traffic FROM stixshifter://idsX WHERE dst_port = 80INFO nt

DISP [TIMESTAMPED(varx)|varx]     [WHERE ecgp]     [ATTR attribute1, attribute2, ...]     [SORT BY attibute [ASC|DESC]]     [LIMIT l [OFFSET n]]

# display <source IP, source port, destination IP, destination port>nt = GET network-traffic FROM stixshifter://idsX WHERE dst_port = 80DISP nt ATTR src_ref.value, src_port, dst_ref.value, dst_port# display process pid, name, and command lineprocs = GET process FROM stixshifter://edrA WHERE parent_ref.name = 'bash'DISP procs ATTR pid, name, command_line# display the timestamps from observations of those processes:DISP TIMESTAMPED(procs) ATTR pid, name, command_line

DESCRIBE varx.attr

# showing information like unique count of src_portnt = GET network-traffic FROM stixshifter://idsX WHERE dst_port = 80DESCRIBE nt.src_port

SAVE varx TO file_path

# save all process records into /tmp/kestrel_procs.parquet.gzprocs = GET process FROM stixshifter://edrA WHERE parent_ref.name = 'bash'SAVE procs TO /tmp/kestrel_procs.parquet.gz

newvar = LOAD file_path [AS entity_type]

# save all process records into /tmp/kestrel_procs.parquet.gzprocs = GET process FROM stixshifter://edrA WHERE parent_ref.name = 'bash'SAVE procs TO /tmp/kestrel_procs.parquet.gz# in another hunt, load the processespload = LOAD /tmp/kestrel_procs.parquet.gz# load suspicious IPs from a threat intelligence source# the file /tmp/suspicious_ips.csv only has one column `value`, which is the IPsusp_ips = LOAD /tmp/suspicious_ips.csv AS ipv4-addr# check whether there is any network-traffic goes to susp_ipsnt = GET network-traffic     FROM stixshifter://idsX     WHERE dst_ref.value = susp_ips.value

newvar = oldvarnewvar = TIMESTAMPED(oldvar)newvar = oldvar [WHERE ecgp] [ATTR attr1,...] [SORT BY attr] [LIMIT n [OFFSET m]]

# copy procscopy_of_procs = procs# filter conns for SSH connectionsssh_conns = conns WHERE dst_port = 22# get URLs with their timestampsts_urls = TIMESTAMPED(urls)# filter procs for WMIC commands with timestampswmic_procs = TIMESTAMPED(procs) WHERE command_line LIKE '%wmic%'# WHERE clause examplesp2 = procs WHERE pid IN (4, 198, 2874)p3 = procs WHERE pid = p2.pidp4 = procs WHERE pid IN (p2.pid, 8888, 10002)p5 = procs WHERE pid = p2.pid AND name = "explorer.exe"

merged_var = var1 + var2 + var3 + ...

# one TTP matchingprocsA = GET process FROM stixshifter://edrA WHERE parent_ref.name = 'bash'# another TTP matchingprocsB = GET process FROM stixshifter://edrA WHERE binary_ref.name = 'sudo'# merge results of bothprocs = procsA + procsB# further hunt flowAPPLY docker://susp_proc_scoring ON procs

newvar = JOIN varA, varB BY attribute1, attribute2

procsA = GET process FROM stixshifter://edrA WHERE name = 'bash'procsB = GET process WHERE binary_ref.name = 'sudo'# get only processes from procsA that have a child process in procsBprocsC = JOIN procsA, procsB BY pid, parent_ref.pid# an alternative way of doing it without knowing the reference attributeprocsD = FIND process CREATED procsBprocsE = GET process FROM procsD WHERE pid = procsA.pid

{   "id": "observed-data--10fb1d74-41f9-4761-a72a-0002",   "type": "observed-data",   "spec_version": "2.1",   "created": "2022-12-08T00:03:19",   "modified": "2022-12-08T00:03:19",   "objects": {      "0": {          "type": "ipv4-addr",          "value": "0.0.4.66"      },      "1": {          "type": "ipv4-addr",          "value": "0.0.4.68"      },      "2": {          "type": "user-account",          "user_id": "C466@DOM1",          "status": "benign"      },      "3": {          "type": "network-traffic",          "src_ref": "0",          "dst_ref": "1"      }    },    "first_observed": "2022-10-05T00:03:19.000Z",    "last_observed": "2022-10-05T00:03:19.000Z",    "number_observed":1}

$ git clone https://github.com/opencybersecurityalliance/kestrel-analytics/lateral-movement-detection.git$ docker build -t kestrel-analytics-detect_lateral_movement

observations = GET observed-data FROM stixshifter://database WHERE [user-account:user_id != null]DISP observations LIMIT 10

users=GET user-account FROM stixshifter://database WHERE [user-account:user_id != null]connections=FIND network-traffic LINKED users

connections_obs = ADDOBSID (connections)users_obs = ADDOBSID (users)

DISP users_obs LIMIT 4

APPLY docker://detect_lateral_movement ON observations, users_obs, connections_obs WITH ku=60, ks=60, kd=60

DISP observations ATTR id, created, first_observed, source, destination, username, status LIMIT 10

users = GET user-account FROM stixshifter://database WHERE [user-account:user_id != null]connections = FIND network-traffic LINKED usersconnections_obs = ADDOBSID (connections)users_obs = ADDOBSID (users)observations = GET observed-data FROM stixshifter://database WHERE [user-account:user_id != null]

$ git clone https://github.com/opencybersecurityalliance/kestrel-analytics/Graph Learning-based Lateral Movement Detection.git$ docker build -t kestrel-analytics-detect_lm

APPLY docker://detect_lm ON observations, users_obs, connections_obs WITH walkLength=3, classifier=xgboost

# should be executed in directory /integrations/elk/sf/make -f Makefile.no_elk run

$ python -m venv huntingspace$ . huntingspace/bin/activate$ pip install -U pip setuptools wheel

# dependent packages such as `stix-shifter` will be automatically installed by `pip`$ pip install kestrel-jupyter$ python -m kestrel_jupyter_kernel.setup

$ pip install stix-shifter-modules-elastic-ecs

$ export STIXSHIFTER_CONTAINER1_CONNECTOR=elastic_ecs$ export STIXSHIFTER_CONTAINER1_CONNECTION='{"host":"elastic.ourcloud.com", "port":9200, "indices":"container1-sysflow-*"}'$ export STIXSHIFTER_CONTAINER1_CONFIG='{"auth":{"id":"VuaCfGcBCdbkQm-e5aOx", "api_key":"ui2lp2axTNmsyakw9tvNnw"}}'

$ export STIXSHIFTER_LAPTOP1_CONNECTOR=elastic_ecs$ export STIXSHIFTER_LAPTOP1_CONNECTION='{"host":"elastic.ourcloud.com", "port":9200, "indices":"employee-laptop-1"}'$ export STIXSHIFTER_LAPTOP1_CONFIG='{"auth":{"id":"VuaCfGcBCdbkQm-e5aOx", "api_key":"ui2lp2axTNmsyakw9tvNnw"}}'

$ jupyter notebook

# get all `ping` processes on Oct 17, 2021 into Kestrel variable `hello`hello = GET process FROM stixshifter://container1        WHERE [process:name = 'ping']        START t'2021-10-17T00:00:00Z' STOP t'2021-10-18T00:00:00Z'

exploits = GET process FROM stixshifter://container1           # The TTP tag simplifies enum `process:name IN ('sh', 'bash', 'csh', ...)`           # https://attack.mitre.org/techniques/T1059/           WHERE [process:parent_ref.name = 'node' AND process:x_ttp_tags LIKE '%T1059%']           START t'2021-10-17T00:00:00Z' STOP t'2021-10-18T00:00:00Z'

# What did the exploited process do,#   e.g., did it execute any commands?#         did it probe the host info?#         did it established a C&C?#         did it do anything harmful?## Let's find child processes of the exploited processes and display them chronicallyexp_spawn = FIND process CREATED BY exploitsexp_act = GET process FROM exp_spawn WHERE [process:name != 'bash']exp_act = SORT exp_act BY created ASCDISP exp_act ATTR created, pid, parent_ref.pid, name, command_line

$ git clone https://github.com/opencybersecurityalliance/kestrel-analytics.git

$ docker build -t kestrel-analytics-susp_scoring

procs = GET process        FROM file://samplestix.json        WHERE [process:parent_ref.name = 'cmd.exe']APPLY docker://susp_scoring ON procs

 [process:parent_ref.name = 'cmd.exe']cmd.exesusp_scoringprocsprocessprocsx_suspicious_score

$ cp -r template/ analytics/my_analytic

$ cd analytics/my_analytic$ docker build -t kestrel-analytics-my_analytic .

APPLY analytics_identifier ON var1, var2, ... WITH x=1, y=abc

import osimport pandas as pdfrom sklearn.cluster import KMeans# Kestrel analytics default paths (single input variable)INPUT_DATA_PATH = "/data/input/0.parquet.gz"OUTPUT_DATA_PATH = "/data/output/0.parquet.gz"# Our analytic parameters from the WITH clause# Kestrel will create env vars for them COLS = os.environ['columns']N = os.environ['n']def analytics(df):    # Process our parameters    cols = COLS.split(',')    n = int(N)    # Run the algorithm    kmeans = KMeans(n_clusters=n).fit(df[cols])    df['cluster'] = kmeans.labels_    # return the updated Kestrel variable    return dfif __name__ == "__main__":    dfi = pd.read_parquet(INPUT_DATA_PATH)    dfo = analytics(dfi)    dfo.to_parquet(OUTPUT_DATA_PATH, compression="gzip")

# Our analytic parameters from the WITH clause# Kestrel will create env vars for them COLS = os.environ['columns']N = os.environ['n']

#################################################################               Kestrel Analytics - scikit-learn clustering## Build the analytics container:#   docker build -t kestrel-analytics-my_analytic .## Call the analytics in Kestrel#   APPLY my_analytic ON my_var WITH n=N,columns=...#################################################################FROM python:3RUN pip install --upgrade pip &&     pip install --no-cache-dir pandas pyarrow scikit-learnWORKDIR /opt/analyticsADD analytics.py .CMD ["python", "analytics.py"]

APPLY docker://my_analytic ON dns_conns WITH n=2, columns=src_byte_count,dst_byte_count