From 61940147ef9c5804f4168dcd57a57cfd12053328 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Wed, 26 Jan 2022 17:02:51 +0000 Subject: [PATCH] Update parser to support all available reports. Update to existing test cases to match current report types. New tests for added report types. pycodestyle fixes add testdata licenses pycodestyle fix Added reports parameter Suggested changes to the parser Proposed details for the release Test script updates for suggested changes Test input updates Realign columns Update compromised_website.csv Update scan_adb.csv Update scan_adb.csv Update scan_ftp.csv Update scan_ipp.csv Update scan_snmp.csv Realign columns Remove duplicates Changed malware.name to extra.infection Updated SPDX-FileCopyrightText shadowserver api: document and warn on old parameter document the old parameter `country` and its status warn if used adapt the test DOC: fix NEWS entry of PR#2143 Added the sector field to scan_amqp, scan_cwmp, and scan_vnc. Copyright and raw field updates Added the sector field to scan_amqp, scan_cwmp, and scan_vnc. Copyright updates Added phish_url and scan_modbus reports. Update source.url and source.fqdn for phish_url and malware_url reports. Update classification.taxonomy and classification.type for scan_modbus report. * additional field type validation changes * added count, bytes, duration, avg_pps, and max_pps fields to event_honeypot_ddos_amp * added 'protocol.application': 'https' to scan_ssl, scan_ssl_freak, and scan_ssl_poodle * added 'extra.tag' to scan_* and device_id Replaced scan_modbus with scan_ics Addeed event4_honeypot_ddos, event4_honeypot_ddos_target, scan_dvr_dhcpdiscover, and scan_socks. Tests for event4_honeypot_ddos. Tests for event4_honeypot_ddos_target. Tests for scan_dvr_dhcpdiscover. Tests for scan_socks. Rename file Rename file update:scan_mdns, scan_smb, and special; add:scan_ddos_middle_box cleanup renamed license files updated scan_mdns test files updated scan_smb test files updated special test files add scan_ddos_middlebox test files add scan_ddos_middlebox test updated schema Updated scan_smb tests Updated scan_ntp tests Updated scan_snmp tests New scan_docker test New scan_kubernetes test New scan_mysql test Updated report schema for June 2022 Added scan_epmd test Revert "Added scan_epmd test" This reverts commit 01edea18d82b583290f66eae5f2457f4695623cd. Revert: Fix for recover_line method as commited in certtools#2192 Added scan_couchdb Test case for scan_couchdb Added scan6_rpd Added/updated README with maintainer details Restored feed names and classification.identifiers to minimize upgrade impact. Merge repair pycodestyle repairs codespell fixes license compliance fixes pycodestyle fixes Feed configuration updates for compatibility with the original. Added scan_postgres test Added additional IPv6 aliases Fix for recover_line method as commited in certtools#2192 --- NEWS.md | 126 + docs/user/bots.rst | 3 +- .../bots/collectors/shadowserver/README.md | 9 + intelmq/bots/parsers/shadowserver/README.md | 9 + intelmq/bots/parsers/shadowserver/_config.py | 4483 ++++++++++------- intelmq/bots/parsers/shadowserver/parser.py | 4 - intelmq/lib/test.py | 19 +- intelmq/lib/upgrades.py | 33 +- .../bots/collectors/shadowserver/README.md | 9 + .../test_collector_reports_api.py | 6 +- .../tests/bots/parsers/shadowserver/README.md | 6 + ...p.csv.license => scan_rdpeudp.csv.license} | 0 .../parsers/shadowserver/test_blocklist.py | 8 +- .../parsers/shadowserver/test_botnet_drone.py | 280 - .../bots/parsers/shadowserver/test_broken.py | 8 +- .../shadowserver/test_caida_ip_spoofer.py | 178 - .../bots/parsers/shadowserver/test_darknet.py | 104 - .../shadowserver/test_ddos_amplification.py | 99 - .../parsers/shadowserver/test_device_id.py | 116 + .../shadowserver/test_drone_brute_force.py | 73 - .../test_event4_honeypot_darknet.py | 9 +- .../shadowserver/test_event4_honeypot_ddos.py | 148 + .../test_event4_honeypot_ddos_target.py | 150 + .../test_event4_honeypot_http_scan.py | 16 +- .../shadowserver/test_event4_ip_spoofer.py | 15 +- .../test_event4_microsoft_sinkhole.py | 13 +- .../test_event4_microsoft_sinkhole_http.py | 16 +- .../shadowserver/test_event4_sinkhole.py | 54 +- .../shadowserver/test_event4_sinkhole_dns.py | 127 + .../shadowserver/test_event4_sinkhole_http.py | 15 +- .../test_event4_sinkhole_http_referer.py | 40 +- .../shadowserver/test_event6_sinkhole_http.py | 146 + .../shadowserver/test_honeypot_brute_force.py | 3 +- .../parsers/shadowserver/test_hp_http_scan.py | 119 - .../parsers/shadowserver/test_hp_ics_scan.py | 117 - .../parsers/shadowserver/test_malware_url.py | 107 + .../shadowserver/test_microsoft_sinkhole.py | 333 -- .../shadowserver/test_outdated_dnssec_key.py | 93 - .../parsers/shadowserver/test_phish_url.py | 106 + .../parsers/shadowserver/test_sandbox_conn.py | 99 + .../parsers/shadowserver/test_sandbox_dns.py | 95 + .../parsers/shadowserver/test_sandbox_url.py | 104 + .../parsers/shadowserver/test_scan_adb.py | 2 +- .../parsers/shadowserver/test_scan_amqp.py | 144 + ...ll.py => test_scan_cisco_smart_install.py} | 4 +- .../parsers/shadowserver/test_scan_couchdb.py | 128 + .../parsers/shadowserver/test_scan_db2.py | 2 + .../shadowserver/test_scan_ddos_middlebox.py | 119 + .../parsers/shadowserver/test_scan_docker.py | 159 + .../test_scan_dvr_dhcpdiscover.py | 179 + .../parsers/shadowserver/test_scan_ftp.py | 2 +- .../parsers/shadowserver/test_scan_http.py | 16 +- .../shadowserver/test_scan_http_vulnerable.py | 10 +- .../parsers/shadowserver/test_scan_ics.py | 125 + .../parsers/shadowserver/test_scan_ipmi.py | 4 - .../parsers/shadowserver/test_scan_isakmp.py | 24 +- .../shadowserver/test_scan_kubernetes.py | 214 + ...est_scan_ldap.py => test_scan_ldap_udp.py} | 8 +- .../parsers/shadowserver/test_scan_mdns.py | 132 +- .../parsers/shadowserver/test_scan_mqtt.py | 95 +- .../shadowserver/test_scan_mqtt_anon.py | 173 + .../parsers/shadowserver/test_scan_mysql.py | 258 + .../parsers/shadowserver/test_scan_netbios.py | 123 + ...is_router.py => test_scan_netis_router.py} | 4 +- .../parsers/shadowserver/test_scan_ntp.py | 182 +- .../shadowserver/test_scan_postgres.py | 199 + .../parsers/shadowserver/test_scan_quic.py | 118 + .../parsers/shadowserver/test_scan_radmin.py | 2 - ...scan_msrdpeudp.py => test_scan_rdpeudp.py} | 4 +- .../parsers/shadowserver/test_scan_smb.py | 116 +- .../shadowserver/test_scan_smb_json.py | 114 +- .../shadowserver/test_scan_smtp_vulnerable.py | 88 +- .../parsers/shadowserver/test_scan_snmp.py | 284 +- .../parsers/shadowserver/test_scan_socks.py | 107 + .../parsers/shadowserver/test_scan_ssdp.py | 87 +- .../parsers/shadowserver/test_scan_ssh.py | 182 + .../parsers/shadowserver/test_scan_ssl.py | 218 + .../shadowserver/test_scan_ssl_freak.py | 8 +- .../shadowserver/test_scan_ssl_poodle.py | 4 +- .../shadowserver/test_scan_synfulknock.py | 117 + .../shadowserver/test_sinkhole6_http.py | 104 - .../parsers/shadowserver/test_sinkhole_dns.py | 91 - .../shadowserver/test_sinkhole_http_drone.py | 92 - .../bots/parsers/shadowserver/test_special.py | 106 + .../shadowserver/testdata/blocklist.csv | 8 +- .../shadowserver/testdata/botnet_drone.csv | 103 - .../testdata/caida_ip_spoofer.csv | 6 - .../testdata/compromised_website.csv | 7 +- .../parsers/shadowserver/testdata/darknet.csv | 134 - .../testdata/ddos_amplification.csv | 3 - .../shadowserver/testdata/device_id.csv | 4 + .../testdata/device_id.csv.license | 2 + .../testdata/drone_brute_force.csv | 2 - .../testdata/event4_honeypot_ddos.csv | 4 + .../testdata/event4_honeypot_ddos.csv.license | 2 + .../testdata/event4_honeypot_ddos_amp.csv | 12 +- .../testdata/event4_honeypot_ddos_target.csv | 4 + .../event4_honeypot_ddos_target.csv.license | 2 + .../testdata/event4_microsoft_sinkhole.csv | 12 +- .../event4_microsoft_sinkhole_http.csv | 10 +- .../shadowserver/testdata/event4_sinkhole.csv | 9 +- .../testdata/event4_sinkhole_dns.csv | 4 + .../testdata/event4_sinkhole_dns.csv.license | 2 + .../testdata/event6_sinkhole_http.csv | 4 + .../testdata/event6_sinkhole_http.csv.license | 2 + .../shadowserver/testdata/hp_http_scan.csv | 3 - .../shadowserver/testdata/hp_ics_scan.csv | 3 - .../shadowserver/testdata/malware_url.csv | 4 + .../testdata/malware_url.csv.license | 2 + .../testdata/microsoft_sinkhole.csv | 108 - .../testdata/outdated_dnssec_key.csv | 3 - .../shadowserver/testdata/phish_url.csv | 4 + .../testdata/phish_url.csv.license | 2 + .../shadowserver/testdata/sandbox_conn.csv | 4 + .../testdata/sandbox_conn.csv.license | 2 + .../shadowserver/testdata/sandbox_dns.csv | 4 + .../testdata/sandbox_dns.csv.license | 2 + .../shadowserver/testdata/sandbox_url.csv | 4 + .../testdata/sandbox_url.csv.license | 2 + .../shadowserver/testdata/scan_adb.csv | 6 +- .../shadowserver/testdata/scan_amqp.csv | 4 + .../testdata/scan_amqp.csv.license | 2 + ...stall.csv => scan_cisco_smart_install.csv} | 0 ...e => scan_cisco_smart_install.csv.license} | 0 .../shadowserver/testdata/scan_coap.csv | 4 +- .../shadowserver/testdata/scan_couchdb.csv | 4 + .../testdata/scan_couchdb.csv.license | 2 + .../shadowserver/testdata/scan_cwmp.csv | 6 +- .../testdata/scan_ddos_middlebox.csv | 4 + .../testdata/scan_ddos_middlebox.csv.license | 2 + .../shadowserver/testdata/scan_docker.csv | 4 + .../testdata/scan_docker.csv.license | 2 + .../testdata/scan_dvr_dhcpdiscover.csv | 4 + .../scan_dvr_dhcpdiscover.csv.license | 2 + .../shadowserver/testdata/scan_ftp.csv | 6 +- .../shadowserver/testdata/scan_ics.csv | 4 + .../testdata/scan_ics.csv.license | 2 + .../shadowserver/testdata/scan_ipp.csv | 4 +- .../shadowserver/testdata/scan_kubernetes.csv | 4 + .../testdata/scan_kubernetes.csv.license | 2 + .../testdata/scan_ldap.csv.license | 2 - .../{scan_ldap.csv => scan_ldap_udp.csv} | 0 ....csv.license => scan_ldap_udp.csv.license} | 0 .../shadowserver/testdata/scan_mdns.csv | 98 +- .../testdata/scan_mdns.csv.license | 2 +- .../shadowserver/testdata/scan_mqtt.csv | 5 +- .../shadowserver/testdata/scan_mqtt_anon.csv | 4 + .../testdata/scan_mqtt_anon.csv.license | 2 + .../shadowserver/testdata/scan_mysql.csv | 4 + .../testdata/scan_mysql.csv.license | 2 + ...netis_router.csv => scan_netis_router.csv} | 0 ....license => scan_netis_router.csv.license} | 0 .../shadowserver/testdata/scan_ntp.csv | 7 +- .../shadowserver/testdata/scan_postgres.csv | 4 + .../testdata/scan_postgres.csv.license | 2 + .../shadowserver/testdata/scan_quic.csv | 4 + .../testdata/scan_quic.csv.license | 2 + .../shadowserver/testdata/scan_rdp.csv | 2 +- .../{scan_msrdpeudp.csv => scan_rdpeudp.csv} | 0 .../testdata/scan_rdpeudp.csv.license | 2 + .../shadowserver/testdata/scan_smb.csv | 7 +- .../testdata/scan_smb.csv.license | 2 +- .../shadowserver/testdata/scan_snmp.csv | 91 +- .../testdata/scan_snmp.csv.license | 2 +- .../shadowserver/testdata/scan_socks.csv | 4 + .../testdata/scan_socks.csv.license | 2 + .../shadowserver/testdata/scan_ssdp.csv | 5 +- .../shadowserver/testdata/scan_ssh.csv | 4 + .../testdata/scan_ssh.csv.license | 2 + .../shadowserver/testdata/scan_ssl.csv | 4 + .../testdata/scan_ssl.csv.license | 2 + .../shadowserver/testdata/scan_ssl_freak.csv | 91 +- .../shadowserver/testdata/scan_ssl_poodle.csv | 63 +- .../testdata/scan_synfulknock.csv | 4 + .../testdata/scan_synfulknock.csv.license | 2 + .../shadowserver/testdata/scan_vnc.csv | 6 +- .../shadowserver/testdata/sinkhole6_http.csv | 4 - .../testdata/sinkhole6_http.csv.license | 2 - .../shadowserver/testdata/sinkhole_dns.csv | 3 - .../testdata/sinkhole_dns.csv.license | 2 - .../testdata/sinkhole_http_drone.csv | 87 - .../parsers/shadowserver/testdata/special.csv | 4 + .../shadowserver/testdata/special.csv.license | 2 + 183 files changed, 7797 insertions(+), 4937 deletions(-) create mode 100644 intelmq/bots/collectors/shadowserver/README.md create mode 100644 intelmq/bots/parsers/shadowserver/README.md create mode 100644 intelmq/tests/bots/collectors/shadowserver/README.md rename intelmq/tests/bots/parsers/shadowserver/{testdata/scan_msrdpeudp.csv.license => scan_rdpeudp.csv.license} (100%) delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_botnet_drone.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_caida_ip_spoofer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_darknet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_ddos_amplification.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_device_id.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_drone_brute_force.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_hp_http_scan.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_hp_ics_scan.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_malware_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_microsoft_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_outdated_dnssec_key.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_phish_url.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py rename intelmq/tests/bots/parsers/shadowserver/{test_cisco_smart_install.py => test_scan_cisco_smart_install.py} (94%) create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py rename intelmq/tests/bots/parsers/shadowserver/{test_scan_ldap.py => test_scan_ldap_udp.py} (97%) create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py rename intelmq/tests/bots/parsers/shadowserver/{test_netis_router.py => test_scan_netis_router.py} (91%) create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py rename intelmq/tests/bots/parsers/shadowserver/{test_scan_msrdpeudp.py => test_scan_rdpeudp.py} (96%) create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sinkhole6_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sinkhole_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sinkhole_http_drone.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_special.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/microsoft_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license rename intelmq/tests/bots/parsers/shadowserver/testdata/{cisco_smart_install.csv => scan_cisco_smart_install.csv} (100%) rename intelmq/tests/bots/parsers/shadowserver/testdata/{cisco_smart_install.csv.license => scan_cisco_smart_install.csv.license} (100%) create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap.csv.license rename intelmq/tests/bots/parsers/shadowserver/testdata/{scan_ldap.csv => scan_ldap_udp.csv} (100%) rename intelmq/tests/bots/parsers/shadowserver/testdata/{microsoft_sinkhole.csv.license => scan_ldap_udp.csv.license} (100%) create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license rename intelmq/tests/bots/parsers/shadowserver/testdata/{netis_router.csv => scan_netis_router.csv} (100%) rename intelmq/tests/bots/parsers/shadowserver/testdata/{netis_router.csv.license => scan_netis_router.csv.license} (100%) create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license rename intelmq/tests/bots/parsers/shadowserver/testdata/{scan_msrdpeudp.csv => scan_rdpeudp.csv} (100%) create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole6_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole6_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license diff --git a/NEWS.md b/NEWS.md index 9fb00620e..c8761bfcc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,11 +9,53 @@ NEWS This file lists all changes which have an affect on the administration of IntelMQ and contains steps that you need to be aware off for the upgrade. Please refer to the changelog for a full list of changes. + 3.1.0 Feature release (unreleased) ---------------------------------- ### Requirements +### Bots +#### ShadowServer Reports API collector +The misleading `country` parameter has been depreciated and a `reports` parameter has been added. +The backwards-compatibility will be removed in IntelMQ version 4.0.0. +See the [Shadowserver Reports API bot's documentation](https://intelmq.readthedocs.io/en/latest/user/bots.html#shadowserver-reports-api). + +#### ShadowServer parser +Previously, mappings used a mix of `extra.naics` and `extra.source.naics`. The parser has been updated to use the more specific term (`extra.source.naics`). + +A number of the _classification.identifier_ values have been updated to follow a common naming convention based on their canonical report name: + +| before IntelMQ 3.1.0 | in IntelMQ 3.1.0 and higher | +| --- | --- | +| accessible-adb | open-adb | +| accessible-afp | open-afp | +| accessible-amqp | open-amqp | +| accessible-ard | open-ard | +| accessible-cisco-smart-install | open-cisco-smart-install | +| accessible-coap | open-coap | +| accessible-ftp | open-ftp | +| accessible-hadoop | open-hadoop | +| accessible-http | open-http | +| accessible-msrdpeudp | open-rdpeudp | +| accessible-radmin | open-radmin | +| accessible-rsync | open-rsync | +| accessible-ubiquiti-discovery-service | open-ubiquiti | +| amplification-ddos-victim | honeypot-ddos-amp | +| blacklisted-ip | blocklist | +| dns-open-resolver | open-dns | +| honeypot-http-scan | honeypot-http-scan | +| ics | honeypot-ics-scan | +| ntp-monitor | open-ntpmonitor | +| ntp-version | open-ntp | +| open-db2-discovery-service | open-db2 | +| open-ike | open-isakmp | +| open-ldap | open-ldap-tcp | +| open-natpmp | open-nat-pmp | +| open-netbios-nameservice | open-netbios | +| open-netis | open-netis-router | +| sinkholedns | sinkhole-dns | + ### Tools ### Data Format @@ -37,6 +79,90 @@ The parameter `timeout` has been merged into `redis_cache_ttl`. ### Libraries ### Postgres databases +The following statements optionally update existing data for the harmonization classification changes: +```sql +UPDATE events + SET "classification.identifier" = 'open-adb' + WHERE "classification.identifier" = 'accessible-adb'; +UPDATE events + SET "classification.identifier" = 'open-afp' + WHERE "classification.identifier" = 'accessible-afp'; +UPDATE events + SET "classification.identifier" = 'open-amqp' + WHERE "classification.identifier" = 'accessible-amqp'; +UPDATE events + SET "classification.identifier" = 'open-ard' + WHERE "classification.identifier" = 'accessible-ard'; +UPDATE events + SET "classification.identifier" = 'open-cisco-smart-install' + WHERE "classification.identifier" = 'accessible-cisco-smart-install'; +UPDATE events + SET "classification.identifier" = 'open-coap' + WHERE "classification.identifier" = 'accessible-coap'; +UPDATE events + SET "classification.identifier" = 'open-ftp' + WHERE "classification.identifier" = 'accessible-ftp'; +UPDATE events + SET "classification.identifier" = 'open-hadoop' + WHERE "classification.identifier" = 'accessible-hadoop'; +UPDATE events + SET "classification.identifier" = 'open-http' + WHERE "classification.identifier" = 'accessible-http'; +UPDATE events + SET "classification.identifier" = 'open-rdpeudp' + WHERE "classification.identifier" = 'accessible-msrdpeudp'; +UPDATE events + SET "classification.identifier" = 'open-radmin' + WHERE "classification.identifier" = 'accessible-radmin'; +UPDATE events + SET "classification.identifier" = 'open-rsync' + WHERE "classification.identifier" = 'accessible-rsync'; +UPDATE events + SET "classification.identifier" = 'open-ubiquiti' + WHERE "classification.identifier" = 'accessible-ubiquiti-discovery-service'; +UPDATE events + SET "classification.identifier" = 'honeypot-ddos-amp' + WHERE "classification.identifier" = 'amplification-ddos-victim'; +UPDATE events + SET "classification.identifier" = 'blocklist' + WHERE "classification.identifier" = 'blacklisted-ip'; +UPDATE events + SET "classification.identifier" = 'open-dns' + WHERE "classification.identifier" = 'dns-open-resolver'; +UPDATE events + SET "classification.identifier" = 'honeypot-http-scan' + WHERE "classification.identifier" = 'honeypot-http-scan'; +UPDATE events + SET "classification.identifier" = 'honeypot-ics-scan' + WHERE "classification.identifier" = 'ics'; +UPDATE events + SET "classification.identifier" = 'open-ntpmonitor' + WHERE "classification.identifier" = 'ntp-monitor'; +UPDATE events + SET "classification.identifier" = 'open-ntp' + WHERE "classification.identifier" = 'ntp-version'; +UPDATE events + SET "classification.identifier" = 'open-db2' + WHERE "classification.identifier" = 'open-db2-discovery-service'; +UPDATE events + SET "classification.identifier" = 'open-isakmp' + WHERE "classification.identifier" = 'open-ike'; +UPDATE events + SET "classification.identifier" = 'open-ldap-tcp' + WHERE "classification.identifier" = 'open-ldap'; +UPDATE events + SET "classification.identifier" = 'open-nat-pmp' + WHERE "classification.identifier" = 'open-natpmp'; +UPDATE events + SET "classification.identifier" = 'open-netbios' + WHERE "classification.identifier" = 'open-netbios-nameservice'; +UPDATE events + SET "classification.identifier" = 'open-netis-router' + WHERE "classification.identifier" = 'open-netis'; +UPDATE events + SET "classification.identifier" = 'sinkhole-dns' + WHERE "classification.identifier" = 'sinkholedns'; +``` ### Bots diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 3091fa956..d4a96c0fc 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -654,9 +654,10 @@ The Cache is required to memorize which files have already been processed (TTL n **Configuration Parameters** -* `country`: The country you want to download the reports for +* `country`: **Deprecated:** The country you want to download the reports for. Will be removed in IntelMQ version 4.0.0, use *reports* instead. * `apikey`: Your Shadowserver API key * `secret`: Your Shadowserver API secret +* `reports`: A list of strings or a comma-separated list of the mailing lists you want to process. * `types`: A list of strings or a string of comma-separated values with the names of report types you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). The possible report types are equivalent to the file names given in the section :ref:`Supported Reports ` of the Shadowserver parser. * **Cache parameters** (see in section :ref:`common-parameters`, the default TTL is set to 10 days) diff --git a/intelmq/bots/collectors/shadowserver/README.md b/intelmq/bots/collectors/shadowserver/README.md new file mode 100644 index 000000000..eb0ddfb4a --- /dev/null +++ b/intelmq/bots/collectors/shadowserver/README.md @@ -0,0 +1,9 @@ + + +This module is maintained by [The Shadowserver Foundation](https://www.shadowserver.org/). + +Please contact intelmq@shadowserver.org with any issues or concerns. + diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md new file mode 100644 index 000000000..eb0ddfb4a --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -0,0 +1,9 @@ + + +This module is maintained by [The Shadowserver Foundation](https://www.shadowserver.org/). + +Please contact intelmq@shadowserver.org with any issues or concerns. + diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 6a4521145..2cf2dfc0f 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -223,1432 +223,1466 @@ def force_base64(value: Optional[str]) -> Optional[str]: return value -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-DB2 -open_db2_discovery_service = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port') - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'db2_hostname', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'servername', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-db2-discovery-service', - } -} +def scan_exchange_taxonomy(field): + if field == 'exchange;webshell': + return 'intrusions' + return 'vulnerable' -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-http-report/ -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-report/ -# -# This mapping is for two feeds as they are the same, so we can use this mapping for -# both :) -# -accessible_vulnerable_http = { + +def scan_exchange_type(field): + if field == 'exchange;webshell': + return 'system-compromise' + return 'infected-system' + + +def scan_exchange_identifier(field): + if field == 'exchange;webshell': + return 'exchange-server-webshell' + return 'vulnerable-exchange-server' + + +# BEGIN CONFGEN + +# https://www.shadowserver.org/what-we-do/network-reporting/blocklist-report/ +blocklist = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port') + ('source.ip', 'ip', validate_ip), ], 'optional_fields': [ - ('protocol.transport', 'protocol'), + ('source.network', 'ip', validate_network), + ('extra.', 'tag', validate_to_none), ('source.reverse_dns', 'hostname'), + ('extra.', 'source', validate_to_none), + ('extra.', 'reason', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'tag'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', invalidate_zero), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'blacklisted-ip', 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - } + 'classification.type': 'blacklist', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-mDNS -open_mdns = { +# https://www.shadowserver.org/what-we-do/network-reporting/compromised-website-report/ +compromised_website = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('protocol.transport', 'protocol'), + ('protocol.application', 'application', validate_to_none), + ('source.url', 'url', convert_http_host_and_url, True), + ('source.fqdn', 'http_host', validate_fqdn), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-mdns' in constant_fields + ('malware.name', 'tag'), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), + ('event_description.text', 'category', validate_to_none), + ('extra.', 'system', validate_to_none), + ('extra.', 'detected_since', validate_to_none), + ('extra.', 'server', validate_to_none), + ('extra.', 'redirect_target', validate_to_none), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'mdns_name', validate_to_none), - ('extra.', 'mdns_ipv4', validate_to_none), - ('extra.', 'mdns_ipv6', validate_to_none), - ('extra.', 'services', validate_to_none), - ('extra.', 'workstation_name', validate_to_none), - ('extra.', 'workstation_ipv4', validate_to_none), - ('extra.', 'workstation_ipv6', validate_to_none), - ('extra.', 'workstation_info', validate_to_none), - ('extra.', 'http_name', validate_to_none), - ('extra.', 'http_ipv4', validate_to_none), - ('extra.', 'http_ipv6', validate_to_none), - ('extra.', 'http_ptr', validate_to_none), - ('extra.', 'http_info', validate_to_none), - ('extra.', 'http_target', validate_to_none), - ('extra.', 'http_port', validate_to_none), + ('extra.', 'sector', validate_to_none), + ('extra.', 'cc_url', validate_to_none), + ('extra.', 'family', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-mdns', - 'protocol.application': 'mdns', - } + 'classification.taxonomy': 'intrusions', + 'classification.type': 'system-compromise', + 'classification.identifier': 'compromised-website', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Chargen -open_chargen = { +# https://www.shadowserver.org/what-we-do/network-reporting/device-identification-report/ +device_id = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-chargen' in constant_fields + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.response_size', 'size', convert_int), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), ('extra.', 'sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-chargen', - 'protocol.application': 'chargen', + 'classification.taxonomy': 'other', + 'classification.type': 'undetermined', + 'classification.identifier': 'device-id', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-TFTP -open_tftp = { +# https://www.shadowserver.org/what-we-do/network-reporting/device-identification-report/ +device_id6 = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-tftp' in constant_fields + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'errorcode', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'errormessage', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-tftp', - 'protocol.application': 'tftp', + 'classification.taxonomy': 'other', + 'classification.type': 'undetermined', + 'classification.identifier': 'device-id', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Sinkhole-HTTP-Drone -# legacy (replaced by event46_sinkhole_http) -sinkhole_http_drone = { +# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-brute-force-events-report/ +event_honeypot_brute_force = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'src_port'), + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('destination.url', 'url', convert_http_host_and_url, True), - ('malware.name', 'type'), - ('user_agent', 'http_agent'), - ('source.tor_node', 'tor', set_tor_node), - ('os.name', 'p0f_genre'), - ('os.version', 'p0f_detail'), - ('source.reverse_dns', 'hostname'), - ('destination.port', 'dst_port'), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_referer', validate_to_none), - ('extra.', 'http_referer_ip', validate_ip), - ('extra.', 'http_referer_asn', convert_int), - ('extra.', 'http_referer_geo', validate_to_none), + ('classification.identifier', 'application'), + ('destination.account', 'username', validate_to_none), + ('extra.', 'tag', validate_to_none), + ('protocol.transport', 'protocol'), + ('source.asn', 'src_asn', invalidate_zero), + ('source.geolocation.cc', 'src_geo'), + ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.source.naics', 'src_naics', invalidate_zero), + ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), ('destination.asn', 'dst_asn', invalidate_zero), ('destination.geolocation.cc', 'dst_geo'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http_referer_naics', validate_to_none), - ('extra.', 'http_referer_sic', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), + ('destination.geolocation.region', 'dst_region'), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('malware.name', 'infection'), + ('extra.', 'family', validate_to_none), ('extra.', 'application', validate_to_none), ('extra.', 'version', validate_to_none), + ('extra.', 'event_id', validate_to_none), + ('extra.', 'service', validate_to_none), + ('extra.', 'start_time', validate_to_none), + ('extra.', 'end_time', validate_to_none), + ('extra.', 'client_version', validate_to_none), + ('extra.', 'password', validate_to_none), + ('extra.', 'payload_url', validate_to_none), + ('extra.', 'payload_md5', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - # classification.identifier will be set to (harmonized) malware name by modify expert - # The feed does not include explicit information on the protocol - # but since it is about HTTP the protocol is always set to 'tcp'. - 'protocol.transport': 'tcp', - 'protocol.application': 'http', + 'classification.taxonomy': 'intrusion-attempts', + 'classification.type': 'brute-force', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Sinkhole6-HTTP-Drone -# legacy (replaced by event46_sinkhole_http) -ipv6_sinkhole_http_drone = { +# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-darknet-events-report/ +event_honeypot_darknet = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip'), - ('source.port', 'src_port') + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ - ('source.asn', 'src_asn'), + ('classification.identifier', 'tag', validate_to_none), + ('extra.', 'tag', validate_to_none), + ('protocol.transport', 'protocol'), + ('source.asn', 'src_asn', invalidate_zero), ('source.geolocation.cc', 'src_geo'), ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.source.naics', 'src_naics', invalidate_zero), + ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), ('destination.asn', 'dst_asn', invalidate_zero), ('destination.geolocation.cc', 'dst_geo'), ('destination.geolocation.region', 'dst_region'), - ('destination.port', 'dst_port'), - ('protocol.transport', 'protocol'), - ('malware.name', 'tag'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'sysdesc', validate_to_none), - ('extra.', 'sysname', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('extra.', 'http_agent', validate_to_none), - ('destination.fqdn', 'http_host'), - ('extra.', 'http_referer', validate_to_none), - ('extra.', 'http_referer_ip', validate_to_none), - ('extra.', 'http_referer_asn', validate_to_none), - ('extra.', 'http_referer_geo', validate_to_none), - ('extra.', 'http_referer_region', validate_to_none), - ('extra.', 'forwarded_by', validate_to_none), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('malware.name', 'infection'), + ('extra.', 'family', validate_to_none), + ('extra.', 'application', validate_to_none), + ('extra.', 'version', validate_to_none), + ('extra.', 'event_id', validate_to_none), + ('extra.', 'count', convert_int), ], 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - # classification.identifier will be set to (harmonized) malware name by modify expert - # The feed does not include explicit information on the protocol - # but since it is about HTTP the protocol is always set to 'tcp'. - 'protocol.transport': 'tcp', + 'classification.taxonomy': 'other', + 'classification.type': 'other', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Microsoft-Sinkhole -# legacy (replaced by event46_sinkhole_http) -microsoft_sinkhole = { +# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-events/ +event_honeypot_ddos = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'src_port'), + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('destination.url', 'url', convert_http_host_and_url, True), - ('malware.name', 'type'), - ('source.tor_node', 'tor', set_tor_node), - ('os.name', 'p0f_genre'), - ('os.version', 'p0f_detail'), - ('source.reverse_dns', 'hostname'), - ('destination.port', 'dst_port'), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_agent', validate_to_none), - ('extra.', 'http_referer', validate_to_none), - ('extra.', 'http_referer_ip', validate_ip), - ('extra.', 'http_referer_asn', convert_int), - ('extra.', 'http_referer_geo', validate_to_none), + ('extra.', 'duration', convert_int), + ('extra.', 'attack_src_port', convert_int), + ('extra.', 'http_usessl', convert_bool), + ('extra.', 'ip_header_seqnum', convert_int), + ('extra.', 'ip_header_ttl', convert_int), + ('extra.', 'number_of_connections', convert_int), + ('extra.', 'packet_length', convert_int), + ('extra.', 'packet_randomized', convert_bool), + ('extra.', 'tag', validate_to_none), + ('protocol.transport', 'protocol'), + ('source.asn', 'src_asn', invalidate_zero), + ('source.geolocation.cc', 'src_geo'), + ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.source.naics', 'src_naics', invalidate_zero), + ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), ('destination.asn', 'dst_asn', invalidate_zero), ('destination.geolocation.cc', 'dst_geo'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http_referer_naics', invalidate_zero), - ('extra.', 'http_referer_sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), + ('destination.geolocation.region', 'dst_region'), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'domain_source', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('malware.name', 'infection'), + ('extra.', 'family', validate_to_none), ('extra.', 'application', validate_to_none), ('extra.', 'version', validate_to_none), + ('extra.', 'event_id', validate_to_none), + ('extra.', 'dst_network', validate_to_none), + ('extra.', 'dst_netmask', validate_to_none), + ('extra.', 'attack', validate_to_none), + ('extra.', 'attack_src_ip', validate_to_none), + ('extra.', 'domain', validate_to_none), + ('extra.', 'domain_transaction_id', validate_to_none), + ('extra.', 'gcip', validate_to_none), + ('extra.', 'http_method', validate_to_none), + ('extra.', 'http_path', validate_to_none), + ('extra.', 'http_postdata', validate_to_none), + ('extra.', 'ip_header_ack', validate_to_none), + ('extra.', 'ip_header_acknum', validate_to_none), + ('extra.', 'ip_header_dont_fragment', validate_to_none), + ('extra.', 'ip_header_fin', validate_to_none), + ('extra.', 'ip_header_identity', validate_to_none), + ('extra.', 'ip_header_psh', validate_to_none), + ('extra.', 'ip_header_rst', validate_to_none), + ('extra.', 'ip_header_syn', validate_to_none), + ('extra.', 'ip_header_tos', validate_to_none), + ('extra.', 'ip_header_urg', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - # classification.identifier will be set to (harmonized) malware name by modify expert - 'protocol.transport': 'tcp', - 'protocol.application': 'http', + 'classification.taxonomy': 'availability', + 'classification.type': 'ddos', + 'classification.identifier': 'honeypot-ddos', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Redis -open_redis = { +# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-amplification-ddos-events-report/ +event_honeypot_ddos_amp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ + ('extra.', 'end_time', convert_date_utc), + ('extra.', 'avg_pps', convert_float), + ('extra.', 'max_pps', convert_float), + ('extra.', 'tag', validate_to_none), ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-redis' in constant_fields + ('source.asn', 'src_asn', invalidate_zero), + ('source.geolocation.cc', 'src_geo'), + ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.source.naics', 'src_naics', invalidate_zero), + ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), + ('destination.asn', 'dst_asn', invalidate_zero), + ('destination.geolocation.cc', 'dst_geo'), + ('destination.geolocation.region', 'dst_region'), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('malware.name', 'infection'), + ('extra.', 'family', validate_to_none), + ('extra.', 'application', validate_to_none), ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'git_sha1', validate_to_none), - ('extra.', 'git_dirty_flag', validate_to_none), - ('extra.', 'build_id', validate_to_none), - ('extra.', 'mode', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'architecture', validate_to_none), - ('extra.', 'multiplexing_api', validate_to_none), - ('extra.', 'gcc_version', validate_to_none), - ('extra.', 'process_id', validate_to_none), - ('extra.', 'run_id', validate_to_none), - ('extra.', 'uptime', validate_to_none), - ('extra.', 'connected_clients', validate_to_none), - ('extra.', 'sector', validate_to_none), + ('extra.', 'event_id', validate_to_none), + ('extra.', 'request', validate_to_none), + ('extra.', 'count', convert_int), + ('extra.', 'bytes', convert_int), + ('extra.', 'duration', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-redis', - 'protocol.application': 'redis', + 'classification.identifier': 'amplification-ddos-victim', + 'classification.taxonomy': 'availability', + 'classification.type': 'ddos', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Portmapper -open_portmapper = { +# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-target-events-report/ +event_honeypot_ddos_target = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ + ('extra.', 'duration', convert_int), + ('extra.', 'attack_src_port', convert_int), + ('extra.', 'http_usessl', convert_bool), + ('extra.', 'ip_header_seqnum', convert_int), + ('extra.', 'ip_header_ttl', convert_int), + ('extra.', 'number_of_connections', convert_int), + ('extra.', 'packet_length', convert_int), + ('extra.', 'packet_randomized', convert_bool), + ('extra.', 'tag', validate_to_none), ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-portmapper' in constant_fields - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'programs', validate_to_none), - ('extra.', 'mountd_port', validate_to_none), - ('extra.', 'exports', validate_to_none), - ('extra.', 'sector', validate_to_none), + ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), + ('destination.asn', 'dst_asn', invalidate_zero), + ('destination.geolocation.cc', 'dst_geo'), + ('destination.geolocation.region', 'dst_region'), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('source.asn', 'src_asn', invalidate_zero), + ('source.geolocation.cc', 'src_geo'), + ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.source.naics', 'src_naics', invalidate_zero), + ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'domain_source', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('malware.name', 'infection'), + ('extra.', 'family', validate_to_none), + ('extra.', 'application', validate_to_none), + ('extra.', 'version', validate_to_none), + ('extra.', 'event_id', validate_to_none), + ('extra.', 'dst_network', validate_to_none), + ('extra.', 'dst_netmask', validate_to_none), + ('extra.', 'attack', validate_to_none), + ('extra.', 'attack_src_ip', validate_to_none), + ('extra.', 'domain', validate_to_none), + ('extra.', 'domain_transaction_id', validate_to_none), + ('extra.', 'gcip', validate_to_none), + ('extra.', 'http_method', validate_to_none), + ('extra.', 'http_path', validate_to_none), + ('extra.', 'http_postdata', validate_to_none), + ('extra.', 'ip_header_ack', validate_to_none), + ('extra.', 'ip_header_acknum', validate_to_none), + ('extra.', 'ip_header_dont_fragment', validate_to_none), + ('extra.', 'ip_header_fin', validate_to_none), + ('extra.', 'ip_header_identity', validate_to_none), + ('extra.', 'ip_header_psh', validate_to_none), + ('extra.', 'ip_header_rst', validate_to_none), + ('extra.', 'ip_header_syn', validate_to_none), + ('extra.', 'ip_header_tos', validate_to_none), + ('extra.', 'ip_header_urg', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-portmapper', - 'protocol.application': 'portmapper', + 'classification.taxonomy': 'availability', + 'classification.type': 'ddos', + 'classification.identifier': 'honeypot-ddos-target', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-IPMI -open_ipmi = { +# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-http-scanner-events/ +event_honeypot_http_scan = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-ipmi' in constant_fields - ('extra.', 'ipmi_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'none_auth', convert_bool), - ('extra.', 'md2_auth', convert_bool), - ('extra.', 'md5_auth', convert_bool), - ('extra.', 'passkey_auth', convert_bool), - ('extra.', 'oem_auth', convert_bool), - ('extra.', 'defaultkg', validate_to_none), - ('extra.', 'permessage_auth', convert_bool), - ('extra.', 'userlevel_auth', convert_bool), - ('extra.', 'usernames', convert_bool), - ('extra.', 'nulluser', convert_bool), - ('extra.', 'anon_login', convert_bool), - ('extra.', 'error', validate_to_none), - ('extra.', 'deviceid', validate_to_none), - ('extra.', 'devicerev', validate_to_none), - ('extra.', 'firmwarerev', validate_to_none), + ('user_agent', 'http_agent', validate_to_none), + ('extra.method', 'http_request_method', validate_to_none), + ('extra.', 'tag', validate_to_none), + ('protocol.transport', 'protocol'), + ('source.asn', 'src_asn', invalidate_zero), + ('source.geolocation.cc', 'src_geo'), + ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.source.naics', 'src_naics', invalidate_zero), + ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), + ('destination.asn', 'dst_asn', invalidate_zero), + ('destination.geolocation.cc', 'dst_geo'), + ('destination.geolocation.region', 'dst_region'), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('malware.name', 'infection'), + ('extra.', 'family', validate_to_none), + ('extra.', 'application', validate_to_none), ('extra.', 'version', validate_to_none), - ('extra.', 'manufacturerid', validate_to_none), - ('extra.', 'manufacturername', validate_to_none), - ('extra.', 'productid', validate_to_none), - ('extra.', 'productname', validate_to_none), + ('extra.', 'event_id', validate_to_none), + ('extra.', 'pattern', validate_to_none), + ('destination.url', 'http_url', convert_http_host_and_url, True), + ('extra.', 'url_scheme', validate_to_none), + ('extra.', 'session_tags', validate_to_none), + ('extra.', 'vulnerability_enum', validate_to_none), + ('extra.', 'vulnerability_id', validate_to_none), + ('extra.', 'vulnerability_class', validate_to_none), + ('extra.', 'vulnerability_score', validate_to_none), + ('extra.', 'vulnerability_severity', validate_to_none), + ('extra.', 'vulnerability_version', validate_to_none), + ('extra.', 'threat_framework', validate_to_none), + ('extra.', 'threat_tactic_id', validate_to_none), + ('extra.', 'threat_technique_id', validate_to_none), + ('extra.', 'target_vendor', validate_to_none), + ('extra.', 'target_product', validate_to_none), + ('extra.', 'target_class', validate_to_none), + ('extra.', 'file_md5', validate_to_none), + ('extra.', 'file_sha256', validate_to_none), + ('extra.', 'request_raw', force_base64), + ('extra.', 'body_raw', force_base64), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ipmi', - 'protocol.application': 'ipmi', - 'protocol.transport': 'udp', + 'classification.taxonomy': 'information-gathering', + 'classification.type': 'scanner', + 'protocol.application': 'http', + 'classification.identifier': 'honeypot-http-scan', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-QOTD -open_qotd = { +# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ics-scanner-events-report/ +event_honeypot_ics_scan = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ + ('extra.', 'tag', validate_to_none), ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-qotd' in constant_fields - ('extra.', 'quote', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), + ('source.asn', 'src_asn', invalidate_zero), + ('source.geolocation.cc', 'src_geo'), + ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.source.naics', 'src_naics', invalidate_zero), + ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), + ('destination.asn', 'dst_asn', invalidate_zero), + ('destination.geolocation.cc', 'dst_geo'), + ('destination.geolocation.region', 'dst_region'), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('malware.name', 'infection'), + ('extra.', 'family', validate_to_none), + ('extra.', 'application', validate_to_none), + ('extra.', 'version', validate_to_none), + ('extra.', 'event_id', validate_to_none), + ('extra.', 'state', validate_to_none), + ('extra.', 'sensor_id', validate_to_none), + ('extra.', 'slave_id', validate_to_none), + ('extra.', 'function_code', validate_to_none), + ('extra.', 'request', validate_to_none), + ('extra.', 'response', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-qotd', - 'protocol.application': 'qotd', + 'classification.identifier': 'ics', + 'classification.taxonomy': 'information-gathering', + 'classification.type': 'scanner', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SSDP -open_ssdp = { +# https://www.shadowserver.org/what-we-do/network-reporting/ip-spoofer-events-report/ +event_ip_spoofer = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ + ('extra.', 'infection', validate_to_none), + ('source.network', 'network', validate_to_none), + ('extra.', 'tag', validate_to_none), ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-ssdp' in constant_fields - ('extra.', 'header', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'systime', validate_to_none), - ('extra.', 'cache_control', validate_to_none), - ('extra.', 'location', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'search_target', validate_to_none), - ('extra.', 'unique_service_name', validate_to_none), - ('extra.', 'host', validate_to_none), - ('extra.', 'nts', validate_to_none), - ('extra.', 'nt', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), + ('source.asn', 'src_asn', invalidate_zero), + ('source.geolocation.cc', 'src_geo'), + ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.source.naics', 'src_naics', invalidate_zero), + ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), + ('destination.asn', 'dst_asn', invalidate_zero), + ('destination.geolocation.cc', 'dst_geo'), + ('destination.geolocation.region', 'dst_region'), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('extra.', 'family', validate_to_none), + ('extra.', 'application', validate_to_none), + ('extra.', 'version', validate_to_none), + ('extra.', 'event_id', validate_to_none), + ('extra.', 'routedspoof', validate_to_none), + ('extra.', 'session', validate_to_none), + ('extra.', 'nat', convert_bool), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ssdp', - 'protocol.application': 'ssdp', + 'classification.taxonomy': 'fraud', + 'classification.type': 'masquerade', + 'classification.identifier': 'ip-spoofer', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SNMP -open_snmp = { +# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-events-report/ +event_sinkhole = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ + ('classification.identifier', 'infection', validate_to_none), + ('malware.name', 'family', validate_to_none), + ('extra.', 'tag', validate_to_none), + ('extra.', 'infection', validate_to_none), ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'sysdesc', validate_to_none), - ('extra.', 'sysname', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', convert_int), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), + ('source.asn', 'src_asn', invalidate_zero), + ('source.geolocation.cc', 'src_geo'), + ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.source.naics', 'src_naics', invalidate_zero), + ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), + ('destination.asn', 'dst_asn', invalidate_zero), + ('destination.geolocation.cc', 'dst_geo'), + ('destination.geolocation.region', 'dst_region'), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('extra.', 'application', validate_to_none), + ('extra.', 'version', validate_to_none), + ('extra.', 'event_id', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-snmp', - 'protocol.application': 'snmp', + 'classification.taxonomy': 'malicious-code', + 'classification.type': 'infected-system', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-MSSQL -open_mssql = { +# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-dns-events-report/ +event_sinkhole_dns = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ + ('extra.naics', 'src_naics', invalidate_zero), + ('extra.sector', 'src_sector', validate_to_none), + ('extra.dns_query_type', 'query_type'), + ('extra.dns_query', 'query'), + ('malware.name', 'family', validate_to_none), + ('extra.', 'tag', validate_to_none), + ('extra.', 'infection', validate_to_none), ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-mssql' in constant_fields - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('source.local_hostname', 'server_name'), - ('extra.', 'instance_name', validate_to_none), - ('extra.', 'tcp_port', convert_int), - ('extra.', 'named_pipe', validate_to_none), - ('extra.', 'response_length', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'sector', validate_to_none), + ('source.asn', 'src_asn', invalidate_zero), + ('source.geolocation.cc', 'src_geo'), + ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('extra.', 'count', convert_int), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-mssql', - 'protocol.application': 'mssql', + 'classification.identifier': 'sinkholedns', + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'protocol.application': 'dns', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-MongoDB -open_mongodb = { +# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-events-report/ +event_sinkhole_http = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'src_ip', validate_ip), + ('source.port', 'src_port', convert_int), ], 'optional_fields': [ + ('classification.identifier', 'tag'), + ('malware.name', 'family', validate_to_none), + ('extra.', 'tag', validate_to_none), + ('extra.', 'infection', validate_to_none), ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-mongodb' in constant_fields + ('source.asn', 'src_asn', invalidate_zero), + ('source.geolocation.cc', 'src_geo'), + ('source.geolocation.region', 'src_region'), + ('source.geolocation.city', 'src_city'), + ('source.reverse_dns', 'src_hostname'), + ('extra.source.naics', 'src_naics', invalidate_zero), + ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), + ('destination.asn', 'dst_asn', invalidate_zero), + ('destination.geolocation.cc', 'dst_geo'), + ('destination.geolocation.region', 'dst_region'), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('extra.', 'application', validate_to_none), ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'gitversion', validate_to_none), - ('extra.', 'sysinfo', validate_to_none), - ('extra.', 'opensslversion', validate_to_none), - ('extra.', 'allocator', validate_to_none), - ('extra.', 'javascriptengine', validate_to_none), - ('extra.', 'bits', validate_to_none), - ('extra.', 'maxbsonobjectsize', validate_to_none), - ('extra.', 'ok', validate_to_none), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'sector', validate_to_none), + ('extra.', 'event_id', validate_to_none), + ('destination.url', 'http_url', convert_http_host_and_url, True), + ('destination.fqdn', 'http_host', validate_fqdn), + ('extra.', 'http_agent', validate_to_none), + ('extra.', 'forwarded_by', validate_to_none), + ('extra.', 'ssl_cipher', validate_to_none), + ('extra.', 'http_referer', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-mongodb', - 'protocol.application': 'mongodb', + 'classification.taxonomy': 'malicious-code', + 'classification.type': 'infected-system', + 'protocol.application': 'http', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NetBIOS -open_netbios_nameservice = { +# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-referer-events-report/ +event_sinkhole_http_referer = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), ], 'optional_fields': [ + ('malware.name', 'family', validate_to_none), + ('extra.', 'tag', validate_to_none), + ('extra.', 'infection', validate_to_none), ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-netbios-nameservice' in constant_fields - ('extra.', 'mac_address', validate_to_none), + ('extra.', 'http_referer_ip', validate_ip), + ('extra.', 'http_referer_port', convert_int), + ('extra.', 'http_referer_asn', invalidate_zero), + ('extra.', 'http_referer_geo', validate_to_none), + ('extra.', 'http_referer_region', validate_to_none), + ('extra.', 'http_referer_city', validate_to_none), + ('extra.', 'http_referer_hostname', validate_to_none), + ('extra.', 'http_referer_naics', invalidate_zero), + ('extra.', 'http_referer_sector', validate_to_none), + ('destination.ip', 'dst_ip', validate_ip), + ('destination.port', 'dst_port', convert_int), + ('destination.asn', 'dst_asn', invalidate_zero), + ('destination.geolocation.cc', 'dst_geo'), + ('destination.geolocation.region', 'dst_region'), + ('destination.geolocation.city', 'dst_city'), + ('destination.reverse_dns', 'dst_hostname', validate_to_none), + ('extra.destination.naics', 'dst_naics', invalidate_zero), + ('extra.destination.sector', 'dst_sector', validate_to_none), + ('extra.', 'public_source', validate_to_none), + ('extra.', 'application', validate_to_none), + ('extra.', 'version', validate_to_none), + ('extra.', 'event_id', validate_to_none), + ('destination.url', 'http_url', convert_http_host_and_url, True), + ('destination.fqdn', 'http_host', validate_fqdn), + ('extra.', 'http_referer', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'classification.identifier': 'sinkhole-http-referer', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/malware-url-report/ +malware_url = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ], + 'optional_fields': [ + ('source.url', 'url', convert_http_host_and_url, True), + ('source.fqdn', 'host', validate_fqdn), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'workgroup', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('source.account', 'username'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('malware.name', 'tag'), + ('extra.', 'source', validate_to_none), + ('malware.hash.sha256', 'sha256', validate_to_none), + ('extra.', 'application', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-netbios-nameservice', - 'protocol.application': 'netbios-nameservice', + 'classification.taxonomy': 'malicious-code', + 'classification.type': 'malware-distribution', + 'classification.identifier': 'malware-url', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Elasticsearch -open_elasticsearch = { +phish_url = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), ], 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-elasticsearch' in constant_fields - ('extra.', 'version', validate_to_none), + ('source.url', 'url', convert_http_host_and_url, True), + ('source.fqdn', 'host', validate_fqdn), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ok', convert_bool), - ('extra.', 'name', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'status', convert_int), - ('extra.', 'build_hash', validate_to_none), - ('extra.', 'build_timestamp', validate_to_none), - ('extra.', 'build_snapshot', convert_bool), - ('extra.', 'lucene_version', validate_to_none), - ('extra.', 'tagline', validate_to_none), - ('extra.', 'sector', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'source', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-elasticsearch', - 'protocol.application': 'elasticsearch', + 'classification.taxonomy': 'fraud', + 'classification.type': 'phishing', + 'classification.identifier': 'phish-url', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/DNS-open-resolvers -dns_open_resolvers = { +# http://www.shadowserver.org/wiki/pmwiki.php/Services/Sandbox-Connection +sandbox_conn = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('destination.fqdn', 'host', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), + ('malware.hash.md5', 'md5', validate_to_none), ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'dns-open-resolver' in constant_fields - ('extra.', 'min_amplification', convert_float), - ('extra.', 'dns_version', validate_to_none), - ('os.name', 'p0f_genre'), - ('os.version', 'p0f_detail'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), + ('extra.', 'bytes_in', validate_to_none), + ('extra.', 'bytes_out', validate_to_none), ], 'constant_fields': { - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'classification.identifier': 'dns-open-resolver', + 'classification.taxonomy': 'malicious-code', + 'classification.type': 'malware-distribution', + 'classification.identifier': 'sandbox-conn', + }, +} + +sandbox_dns = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ], + 'optional_fields': [ + ('extra.dns_query_type', 'type', validate_to_none), + ('malware.hash.md5', 'md5hash', validate_to_none), + ('extra.', 'request', validate_to_none), + ('extra.', 'response', validate_to_none), + ('extra.', 'family', validate_to_none), + ('malware.name', 'tag'), + ('extra.', 'source', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'other', + 'classification.type': 'other', 'protocol.application': 'dns', + 'classification.identifier': 'sandbox-dns', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/sandbox-url-report/ +sandbox_url = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ], + 'optional_fields': [ + ('destination.url', 'url', validate_to_none), + ('destination.fqdn', 'host', validate_to_none), + ('extra.http_request_method', 'method', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('malware.hash.md5', 'md5', validate_to_none), + ('user_agent', 'user_agent', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'malicious-code', + 'classification.type': 'malware-distribution', + 'classification.identifier': 'sandbox-url', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/NTP-Monitor -ntp_monitor = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-adb-report/ +scan_adb = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - ('extra.', 'packets', convert_int), - ('extra.', 'size', convert_int), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), + ('extra.', 'name', validate_to_none), + ('extra.', 'model', validate_to_none), + ('extra.', 'device', validate_to_none), + ('extra.', 'features', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('extra.', 'device_version', validate_to_none), + ('extra.', 'device_sector', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'accessible-adb', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'ntp-monitor', - 'protocol.application': 'ntp', + 'protocol.application': 'adb', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Freak-Scan -ssl_freak_vulnerable_servers = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-afp-report/ +scan_afp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'ssl-freak' in constant_fields - ('extra.', 'handshake', validate_to_none), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', validate_to_none), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', validate_to_none), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), + ('extra.', 'machine_type', validate_to_none), + ('extra.', 'afp_versions', validate_to_none), + ('extra.', 'uams', validate_to_none), + ('extra.', 'flags', validate_to_none), + ('extra.', 'server_name', validate_to_none), + ('extra.', 'signature', validate_to_none), + ('extra.', 'directory_service', validate_to_none), + ('extra.', 'utf8_servername', validate_to_none), + ('extra.', 'network_address', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'accessible-afp', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'ssl-freak', - 'protocol.application': 'https', + 'protocol.application': 'afp', }, } -# https://www.shadowserver.org/what-we-do/network-reporting/ssl-poodle-report/ -ssl_poodle46_vulnerable_servers = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ +scan_amqp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'ssl-poodle' in constant_fields - ('extra.', 'handshake', validate_to_none), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', validate_to_none), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', validate_to_none), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), + ('extra.', 'channel', validate_to_none), + ('extra.', 'message_length', validate_to_none), + ('extra.', 'class', validate_to_none), + ('extra.', 'method', validate_to_none), + ('extra.', 'version_major', validate_to_none), + ('extra.', 'version_minor', validate_to_none), + ('extra.', 'capabilities', validate_to_none), + ('extra.', 'cluster_name', validate_to_none), + ('extra.', 'platform', validate_to_none), + ('extra.', 'product', validate_to_none), + ('extra.', 'product_version', validate_to_none), + ('extra.', 'mechanisms', validate_to_none), + ('extra.', 'locales', validate_to_none), ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', validate_to_none), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'accessible-amqp', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'ssl-poodle', - 'protocol.application': 'https', + 'protocol.application': 'amqp', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Memcached -open_memcached = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-apple-remote-desktop-ard-report/ +scan_ard = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-memcached' in constant_fields - ('extra.', 'version', validate_to_none), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'pid', convert_int), - ('extra.', 'pointer_size', convert_int), - ('extra.', 'uptime', convert_int), - ('extra.', 'time', validate_to_none), - ('extra.', 'curr_connections', convert_int), - ('extra.', 'total_connections', convert_int), - ('extra.', 'sector', validate_to_none), + ('extra.', 'machine_name', validate_to_none), + ('extra.', 'response_size', convert_int), ], 'constant_fields': { + 'classification.identifier': 'accessible-ard', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-memcached', - 'protocol.application': 'memcached', }, } -# https://www.shadowserver.org/what-we-do/network-reporting/drone-botnet-drone-report/ -# legacy (replaced by event4_sinkhole, event4_honeypot_darknet and event46_sinkhole_http) -drone = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-chargen-report/ +scan_chargen = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('extra.response_size', 'size', convert_int), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('protocol.transport', 'type'), - ('malware.name', 'infection'), - ('destination.url', 'url', convert_http_host_and_url, True), - ('user_agent', 'agent'), - ('destination.ip', 'cc_ip', validate_ip), - ('destination.port', 'cc_port'), - ('destination.asn', 'cc_asn', invalidate_zero), - ('destination.geolocation.cc', 'cc_geo'), - ('destination.fqdn', 'cc_dns', validate_fqdn), - ('connection_count', 'count', convert_int), - ('extra.', 'proxy', convert_bool), - ('protocol.application', 'application'), - ('os.name', 'p0f_genre'), - ('os.version', 'p0f_detail'), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'id', validate_to_none), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.destination.naics', 'cc_naics', invalidate_zero), - ('extra.destination.sic', 'cc_sic', invalidate_zero), - ('extra.destination.sector', 'cc_sector', validate_to_none), ('extra.', 'sector', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), - ('extra.', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'public_source', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - # classification.identifier will be set to (harmonized) malware name by modify expert + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'chargen', + 'classification.identifier': 'open-chargen', }, } -drone_spam = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-cisco-smart-install-report/ +scan_cisco_smart_install = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('source.fqdn', 'hostname'), - ('protocol.transport', 'type'), - (False, 'infection'), # is just 'spam' - ('source.url', 'url', convert_http_host_and_url, True), - ('user_agent', 'agent'), - ('destination.ip', 'cc_ip', validate_ip), - ('destination.port', 'cc_port'), - ('destination.asn', 'cc_asn', invalidate_zero), - ('destination.geolocation.cc', 'cc_geo'), - ('destination.fqdn', 'cc_dns', validate_fqdn), - ('connection_count', 'count', convert_int), - ('extra.', 'proxy', convert_bool), - ('protocol.application', 'application'), - ('os.name', 'p0f_genre'), - ('os.version', 'p0f_detail'), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'id', validate_to_none), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.destination.naics', 'cc_naics', invalidate_zero), - ('extra.destination.sic', 'cc_sic', invalidate_zero), - ('extra.destination.sector', 'cc_sector', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), - ('extra.', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'public_source', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'abusive-content', - 'classification.type': 'spam', - 'classification.identifier': 'spam', + 'classification.identifier': 'accessible-cisco-smart-install', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'cisco-smart-install', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-XDMCP -open_xdmcp = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-coap-report/ +scan_coap = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-xdmcp' in constant_fields + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'reported_hostname', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'size', convert_int), + ('extra.', 'response', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'accessible-coap', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-xdmcp', - 'protocol.application': 'xdmcp', + 'protocol.application': 'coap', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Compromised-Website -compromised_website = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-couchdb-report/ +scan_couchdb = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('protocol.application', 'application'), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'http_host', validate_fqdn), - ('event_description.text', 'category'), - ('extra.', 'system', validate_to_none), - ('extra.', 'detected_since', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'redirect_target', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'server_version', validate_to_none), + ('extra.', 'couchdb_message', validate_to_none), + ('extra.', 'couchdb_version', validate_to_none), + ('extra.', 'git_sha', validate_to_none), + ('extra.', 'features', validate_to_none), + ('extra.', 'vendor', validate_to_none), + ('extra.', 'visible_databases', validate_to_none), + ('extra.', 'error', validate_to_none), + ('extra.', 'error_reason', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'protocol.application': 'CouchDB', + 'classification.identifier': 'open-couchdb', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NATPMP -open_natpmp = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-cwmp-report/ +scan_cwmp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-natpmp' in constant_fields - ('extra.', 'version', validate_to_none), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'uptime', validate_to_none), - ('extra.', 'external_ip', validate_ip), + ('extra.', 'http', validate_to_none), + ('extra.', 'http_code', convert_int), + ('extra.', 'http_reason', validate_to_none), + ('extra.', 'content_type', validate_to_none), + ('extra.', 'connection', validate_to_none), + ('extra.', 'www_authenticate', validate_to_none), + ('extra.', 'set_cookie', validate_to_none), + ('extra.', 'server', validate_to_none), + ('extra.', 'content_length', convert_int), + ('extra.', 'transfer_encoding', validate_to_none), + ('extra.', 'date', validate_to_none), ('extra.', 'sector', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-natpmp', - 'protocol.application': 'natpmp', + 'protocol.application': 'cwmp', + 'classification.identifier': 'open-cwmp', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Netis-Router -open_netis = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-db2-discovery-service-report/ +scan_db2 = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('extra.', 'size', convert_int), + ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-netis' in constant_fields - ('extra.', 'response', validate_to_none), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), + ('extra.', 'db2_hostname', validate_to_none), + ('extra.', 'servername', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'open-db2-discovery-service', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-netis', - 'protocol.transport': 'udp', + 'protocol.application': 'db2', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/NTP-Version -ntp_version = { +# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-ddos-middlebox-report/ +scan_ddos_middlebox = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('protocol.application', 'tag'), ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'clk_wander', convert_float), - ('extra.', 'clock', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'frequency', convert_float), - ('extra.', 'jitter', convert_float), - ('extra.', 'leap', convert_int), - ('extra.', 'mintc', validate_to_none), - ('extra.', 'noise', convert_float), - ('extra.', 'offset', convert_float), - ('extra.', 'peer', convert_int), - ('extra.', 'phase', convert_float), - ('extra.', 'poll', convert_int), - ('extra.', 'precision', convert_int), - ('extra.', 'processor', validate_to_none), - ('extra.', 'refid', validate_to_none), - ('extra.', 'reftime', validate_to_none), - ('extra.', 'rootdelay', convert_float), - ('extra.', 'rootdispersion', convert_float), - ('extra.', 'stability', convert_float), - ('extra.', 'state', convert_int), - ('extra.', 'stratum', convert_int), - ('extra.', 'system', validate_to_none), - ('extra.', 'tai', convert_int), - ('extra.', 'tc', convert_int), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'source_port', validate_to_none), + ('extra.', 'bytes', convert_int), + ('extra.', 'amplification', convert_float), + ('extra.', 'method', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'ntp-version', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Sandbox-URL -sandbox_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ], - 'optional_fields': [ - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5hash'), - ('source.url', 'url'), - ('user_agent', 'user_agent', validate_to_none), - ('source.fqdn', 'host', validate_fqdn), - ('extra.', 'method', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-url', + 'classification.identifier': 'open-ddos-middlebox', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Spam-URL -spam_url = { +# http://dnsscan.shadowserver.org +scan_dns = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('source.url', 'url'), - ('source.reverse_dns', 'host'), + ('extra.', 'min_amplification', convert_float), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'dns_version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'subject', validate_to_none), - ('extra.', 'ip', validate_ip), - ('extra.', 'src_asn', convert_int), - ('extra.', 'src_geo', validate_to_none), - ('extra.', 'src_region', validate_to_none), - ('extra.', 'src_city', validate_to_none), - ('extra.', 'sender', validate_to_none), + ('os.name', 'p0f_genre'), + ('os.version', 'p0f_detail'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'abusive-content', - 'classification.type': 'spam', - 'classification.identifier': 'spam-url', + 'classification.identifier': 'dns-open-resolver', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'dns', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Vulnerable-ISAKMP -vulnerable_isakmp = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-docker-service-report/ +scan_docker = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-ike' in constant_fields + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'initiator_spi', validate_to_none), - ('extra.', 'responder_spi', validate_to_none), - ('extra.', 'next_payload', convert_int), - ('extra.', 'exchange_type', convert_int), - ('extra.', 'flags', convert_int), - ('extra.', 'message_id', validate_to_none), - ('extra.', 'next_payload2', convert_int), - ('extra.', 'domain_of_interpretation', convert_int), - ('extra.', 'protocol_id', convert_int), - ('extra.', 'spi_size', convert_int), - ('extra.', 'notify_message_type', convert_int), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'http', validate_to_none), + ('extra.', 'http_code', convert_int), + ('extra.', 'http_reason', validate_to_none), + ('extra.', 'content_type', validate_to_none), + ('extra.', 'server', validate_to_none), + ('extra.', 'date', validate_to_none), + ('extra.', 'experimental', validate_to_none), + ('extra.', 'api_version', validate_to_none), + ('extra.', 'arch', validate_to_none), + ('extra.', 'go_version', validate_to_none), + ('extra.', 'os', validate_to_none), + ('extra.', 'kernel_version', validate_to_none), + ('extra.', 'git_commit', validate_to_none), + ('extra.', 'min_api_version', validate_to_none), + ('extra.', 'build_time', validate_to_none), + ('extra.', 'pkg_version', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ike', - 'protocol.application': 'ipsec', - } + 'protocol.application': 'docker', + 'classification.identifier': 'open-docker', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Accessible-RDP -accessible_rdp = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-dvr-dhcpdiscover-report/ +scan_dvr_dhcpdiscover = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('protocol.application', 'tag'), + ('extra.', 'video_input_channels', convert_int), + ('extra.', 'alarm_input_channels', convert_int), + ('extra.', 'video_output_channels', convert_int), + ('extra.', 'alarm_output_channels', convert_int), + ('extra.', 'remote_video_input_channels', convert_int), + ('extra.', 'ipv4_dhcp_enable', convert_bool), + ('extra.', 'ipv6_dhcp_enable', convert_bool), + ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-rdp' in constant_fields - ('extra.', 'handshake', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'rdp_protocol', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', invalidate_zero), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), # always empty so far - ('extra.', 'tlsv13_cipher', validate_to_none), # always empty so far - ('extra.', 'cve20190708_vulnerable', convert_bool), - ('extra.', 'bluekeep_vulnerable', convert_bool), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('extra.', 'device_version', validate_to_none), + ('extra.', 'device_id', validate_to_none), + ('extra.', 'device_serial', validate_to_none), + ('extra.', 'machine_name', validate_to_none), + ('extra.', 'manufacturer', validate_to_none), + ('extra.', 'method', validate_to_none), + ('extra.', 'http_port', convert_int), + ('extra.', 'internal_port', convert_int), + ('extra.', 'mac_address', validate_to_none), + ('extra.', 'ipv4_address', validate_to_none), + ('extra.', 'ipv4_gateway', validate_to_none), + ('extra.', 'ipv4_subnet_mask', validate_to_none), + ('extra.', 'ipv6_address', validate_to_none), + ('extra.', 'ipv6_link_local', validate_to_none), + ('extra.', 'ipv6_gateway', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-rdp', - 'protocol.transport': 'tcp', - 'protocol.application': 'rdp', + 'classification.identifier': 'open-dvr-dhcpdiscover', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Accessible-SMB -accessible_smb = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-elasticsearch-report/ +scan_elasticsearch = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('extra.', 'ok', convert_bool), + ('extra.', 'status', convert_int), + ('extra.', 'build_snapshot', convert_bool), + ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'smb_implant', convert_bool), - ('extra.', 'arch', validate_to_none), - ('extra.', 'key', validate_to_none), + ('extra.', 'name', validate_to_none), + ('extra.', 'cluster_name', validate_to_none), + ('extra.', 'build_hash', validate_to_none), + ('extra.', 'build_timestamp', validate_to_none), + ('extra.', 'lucene_version', validate_to_none), + ('extra.', 'tagline', validate_to_none), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-smb', - 'protocol.transport': 'tcp', - 'protocol.application': 'smb', + 'protocol.application': 'elasticsearch', + 'classification.identifier': 'open-elasticsearch', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-LDAP -open_ldap = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-erlang-port-mapper-report-daemon/ +scan_epmd = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-ldap' in constant_fields + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('source.local_hostname', 'dns_host_name'), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'nodes', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ldap', - 'protocol.application': 'ldap', - } + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'protocol.application': 'Erlang Port Mapper Daemon', + 'classification.identifier': 'open-epmd', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/blocklist-report/ -blocklist = { +# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-exchange-server-report/ +scan_exchange = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('source.ip', 'ip', validate_ip), - ('source.network', 'ip', validate_network), + ('classification.taxonomy', 'tag', scan_exchange_taxonomy), + ('classification.type', 'tag', scan_exchange_type), + ('classification.identifier', 'tag', scan_exchange_identifier), + ('extra.', 'tag', validate_to_none), ('source.reverse_dns', 'hostname'), - ('extra.', 'source', validate_to_none), - ('extra.', 'reason', validate_to_none), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'version', validate_to_none), + ('extra.', 'servername', validate_to_none), + ('extra.', 'url', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'blacklist', - 'classification.identifier': 'blacklisted-ip', - } + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Accessible-Telnet -accessible_telnet = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ftp-report/ +scan_ftp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-telnet' in constant_fields + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), @@ -1656,510 +1690,731 @@ def force_base64(value: Optional[str]) -> Optional[str]: ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), ('extra.', 'banner', validate_to_none), + ('extra.', 'handshake', validate_to_none), + ('extra.', 'cipher_suite', validate_to_none), + ('extra.', 'cert_length', convert_int), + ('extra.', 'subject_common_name', validate_to_none), + ('extra.', 'issuer_common_name', validate_to_none), + ('extra.', 'cert_issue_date', validate_to_none), + ('extra.', 'cert_expiration_date', validate_to_none), + ('extra.', 'sha1_fingerprint', validate_to_none), + ('extra.', 'cert_serial_number', validate_to_none), + ('extra.', 'ssl_version', convert_int), + ('extra.', 'signature_algorithm', validate_to_none), + ('extra.', 'key_algorithm', validate_to_none), + ('extra.', 'subject_organization_name', validate_to_none), + ('extra.', 'subject_organization_unit_name', validate_to_none), + ('extra.', 'subject_country', validate_to_none), + ('extra.', 'subject_state_or_province_name', validate_to_none), + ('extra.', 'subject_locality_name', validate_to_none), + ('extra.', 'subject_street_address', validate_to_none), + ('extra.', 'subject_postal_code', validate_to_none), + ('extra.', 'subject_surname', validate_to_none), + ('extra.', 'subject_given_name', validate_to_none), + ('extra.', 'subject_email_address', validate_to_none), + ('extra.', 'subject_business_category', validate_to_none), + ('extra.', 'subject_serial_number', validate_to_none), + ('extra.', 'issuer_organization_name', validate_to_none), + ('extra.', 'issuer_organization_unit_name', validate_to_none), + ('extra.', 'issuer_country', validate_to_none), + ('extra.', 'issuer_state_or_province_name', validate_to_none), + ('extra.', 'issuer_locality_name', validate_to_none), + ('extra.', 'issuer_street_address', validate_to_none), + ('extra.', 'issuer_postal_code', validate_to_none), + ('extra.', 'issuer_surname', validate_to_none), + ('extra.', 'issuer_given_name', validate_to_none), + ('extra.', 'issuer_email_address', validate_to_none), + ('extra.', 'issuer_business_category', validate_to_none), + ('extra.', 'issuer_serial_number', validate_to_none), + ('extra.', 'sha256_fingerprint', validate_to_none), + ('extra.', 'sha512_fingerprint', validate_to_none), + ('extra.', 'md5_fingerprint', validate_to_none), + ('extra.', 'cert_valid', convert_bool), + ('extra.', 'self_signed', convert_bool), + ('extra.', 'cert_expired', convert_bool), + ('extra.', 'validation_level', validate_to_none), + ('extra.', 'auth_tls_response', validate_to_none), + ('extra.', 'auth_ssl_response', validate_to_none), + ('extra.', 'tlsv13_support', validate_to_none), + ('extra.', 'tlsv13_cipher', validate_to_none), + ('extra.', 'jarm', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('extra.', 'device_version', validate_to_none), + ('extra.', 'device_sector', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'accessible-ftp', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-telnet', - 'protocol.application': 'telnet', - } + 'protocol.application': 'ftp', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-CWMP -accessible_cwmp = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-hadoop-report/ +scan_hadoop = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('protocol.transport', 'protocol'), + ('extra.', 'total_disk', convert_int), + ('extra.', 'used_disk', convert_int), + ('extra.', 'free_disk', convert_int), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-cwmp' in constant_fields + ('extra.', 'version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'date', validate_to_none), + ('extra.', 'server_type', validate_to_none), + ('extra.', 'clusterid', validate_to_none), + ('extra.', 'livenodes', validate_to_none), + ('extra.', 'namenodeaddress', validate_to_none), + ('extra.', 'volumeinfo', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'accessible-hadoop', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-cwmp', - 'protocol.application': 'cwmp', - } + 'protocol.application': 'hadoop', + 'protocol.transport': 'tcp', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Accessible-VNC -accessible_vnc = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-report/ +scan_http = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'product', validate_to_none), - ('extra.', 'banner', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.', 'http', validate_to_none), + ('extra.', 'http_code', convert_int), + ('extra.', 'http_reason', validate_to_none), + ('extra.', 'content_type', validate_to_none), + ('extra.', 'connection', validate_to_none), + ('extra.', 'www_authenticate', validate_to_none), + ('extra.', 'set_cookie', validate_to_none), + ('extra.', 'server', validate_to_none), + ('extra.', 'content_length', convert_int), + ('extra.', 'transfer_encoding', validate_to_none), + ('extra.', 'http_date', convert_date), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-vnc', - 'protocol.transport': 'tcp', - 'protocol.application': 'vnc', - } + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'protocol.application': 'http', + 'classification.identifier': 'open-http', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Accessible-CiscoSmartInstall -accessible_cisco_smart_install = { +# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-http-report/ +scan_http_vulnerable = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'accessible-cisco-smart-install' in constant_fields + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), + ('extra.', 'http', validate_to_none), + ('extra.', 'http_code', convert_int), + ('extra.', 'http_reason', validate_to_none), + ('extra.', 'content_type', validate_to_none), + ('extra.', 'connection', validate_to_none), + ('extra.', 'www_authenticate', validate_to_none), + ('extra.', 'set_cookie', validate_to_none), + ('extra.', 'server', validate_to_none), + ('extra.', 'content_length', convert_int), + ('extra.', 'transfer_encoding', validate_to_none), + ('extra.', 'http_date', convert_date), ], 'constant_fields': { + 'classification.identifier': 'accessible-http', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-cisco-smart-install', - 'protocol.application': 'cisco-smart-install', - } + 'protocol.application': 'http', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Drone-BruteForce -# legacy (replaced by honeypot_brute_force) -drone_brute_force = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ics-report/ +scan_ics = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('protocol.application', 'tag'), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('destination.ip', 'dest_ip', validate_ip), - ('destination.port', 'dest_port'), - ('destination.asn', 'dest_asn', invalidate_zero), - ('destination.geolocation.cc', 'dest_geo'), - ('destination.fqdn', 'dest_dns'), - ('protocol.application', 'service'), - ('classification.identifier', 'service'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.destination.naics', 'dest_naics', invalidate_zero), - ('extra.destination.sic', 'dest_sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.destination.sector', 'dest_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'start_time', validate_to_none), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'client_version', validate_to_none), - ('destination.account', 'username', validate_to_none), - ('extra.', 'password', validate_to_none), - ('extra.', 'payload_url', validate_to_none), - ('extra.', 'payload_md5', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('extra.', 'device_version', validate_to_none), + ('extra.', 'device_id', validate_to_none), + ('extra.', 'response_length', convert_int), + ('extra.', 'raw_response', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - } + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'classification.identifier': 'open-ics', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Accessible-Hadoop -accessible_hadoop = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-ipmi-report/ +scan_ipmi = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('extra.', 'none_auth', convert_bool), + ('extra.', 'md2_auth', convert_bool), + ('extra.', 'md5_auth', convert_bool), + ('extra.', 'passkey_auth', convert_bool), + ('extra.', 'oem_auth', convert_bool), + ('extra.', 'permessage_auth', convert_bool), + ('extra.', 'userlevel_auth', convert_bool), + ('extra.', 'usernames', convert_bool), + ('extra.', 'nulluser', convert_bool), + ('extra.', 'anon_login', convert_bool), ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'ipmi_version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), + ('extra.', 'defaultkg', validate_to_none), + ('extra.', 'error', validate_to_none), + ('extra.', 'deviceid', validate_to_none), + ('extra.', 'devicerev', validate_to_none), + ('extra.', 'firmwarerev', validate_to_none), ('extra.', 'version', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'clusterid', validate_to_none), - ('extra.', 'total_disk', invalidate_zero), - ('extra.', 'used_disk', invalidate_zero), - ('extra.', 'free_disk', invalidate_zero), - ('extra.', 'livenodes', validate_to_none), - ('extra.', 'namenodeaddress', validate_to_none), - ('extra.', 'volumeinfo', validate_to_none), + ('extra.', 'manufacturerid', validate_to_none), + ('extra.', 'manufacturername', validate_to_none), + ('extra.', 'productid', validate_to_none), + ('extra.', 'productname', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), ], 'constant_fields': { - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - } + 'protocol.application': 'ipmi', + 'protocol.transport': 'udp', + 'classification.identifier': 'open-ipmi', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Accessible-ADB -accessible_adb = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-ipp-report/ +scan_ipp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'accessible-adb' in constant_fields + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'name', validate_to_none), - ('extra.', 'model', validate_to_none), - ('extra.', 'device', validate_to_none), - ('extra.', 'features', validate_to_none), + ('extra.', 'ipp_version', validate_to_none), + ('extra.', 'cups_version', validate_to_none), + ('extra.', 'printer_uris', validate_to_none), + ('extra.', 'printer_name', validate_to_none), + ('extra.', 'printer_info', validate_to_none), + ('extra.', 'printer_more_info', validate_to_none), + ('extra.', 'printer_make_and_model', validate_to_none), + ('extra.', 'printer_firmware_name', validate_to_none), + ('extra.', 'printer_firmware_string_version', validate_to_none), + ('extra.', 'printer_firmware_version', validate_to_none), + ('extra.', 'printer_organization', validate_to_none), + ('extra.', 'printer_organization_unit', validate_to_none), + ('extra.', 'printer_uuid', validate_to_none), + ('extra.', 'printer_wifi_ssid', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('extra.', 'device_version', validate_to_none), + ('extra.', 'device_sector', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', + 'protocol.application': 'ipp', + 'classification.identifier': 'open-ipp', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Outdated-DNSSEC-Key -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Outdated-DNSSEC-Key-IPv6 -outdated_dnssec_key = { +# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-isakmp-report/ +scan_isakmp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('extra.', 'spi_size', convert_int), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sic', 'dst_sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.destination.sector', 'dst_sector', validate_to_none), - # ('classification.identifier', 'tag'), # always set to 'outdated-dnssec-key' in constant_fields - ('extra.', 'public_source', validate_to_none), - ('protocol.transport', 'protocol'), + ('extra.', 'initiator_spi', validate_to_none), + ('extra.', 'responder_spi', validate_to_none), + ('extra.', 'next_payload', validate_to_none), + ('extra.', 'exchange_type', validate_to_none), + ('extra.', 'flags', validate_to_none), + ('extra.', 'message_id', validate_to_none), + ('extra.', 'next_payload2', validate_to_none), + ('extra.', 'domain_of_interpretation', validate_to_none), + ('extra.', 'protocol_id', validate_to_none), + ('extra.', 'notify_message_type', validate_to_none), ], 'constant_fields': { - 'protocol.application': 'dns', - 'classification.taxonomy': 'availability', - 'classification.type': 'other', # change to "misconfiguration" when available - 'classification.identifier': 'outdated-dnssec-key', - } + 'classification.identifier': 'open-ike', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'ipsec', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Accessible-rsync -accessible_rsync = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-kubernetes-api-server-report/ +scan_kubernetes = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'accessible-rsync' in constant_fields + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'module', validate_to_none), - ('extra.', 'motd', validate_to_none), - ('extra.', 'password', convert_bool), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'http', validate_to_none), + ('extra.', 'http_code', convert_int), + ('extra.', 'http_reason', validate_to_none), + ('extra.', 'content_type', validate_to_none), + ('extra.', 'server', validate_to_none), + ('extra.', 'date', validate_to_none), + ('extra.', 'major', validate_to_none), + ('extra.', 'minor', validate_to_none), + ('extra.', 'git_version', validate_to_none), + ('extra.', 'git_commit', validate_to_none), + ('extra.', 'git_tree_state', validate_to_none), + ('extra.', 'build_date', validate_to_none), + ('extra.', 'go_version', validate_to_none), + ('extra.', 'compiler', validate_to_none), + ('extra.', 'platform', validate_to_none), + ('extra.', 'handshake', validate_to_none), + ('extra.', 'cipher_suite', validate_to_none), + ('extra.', 'cert_length', convert_int), + ('extra.', 'subject_common_name', validate_to_none), + ('extra.', 'issuer_common_name', validate_to_none), + ('extra.', 'cert_issue_date', validate_to_none), + ('extra.', 'cert_expiration_date', validate_to_none), + ('extra.', 'sha1_fingerprint', validate_to_none), + ('extra.', 'cert_serial_number', validate_to_none), + ('extra.', 'ssl_version', convert_int), + ('extra.', 'signature_algorithm', validate_to_none), + ('extra.', 'key_algorithm', validate_to_none), + ('extra.', 'subject_organization_name', validate_to_none), + ('extra.', 'subject_organization_unit_name', validate_to_none), + ('extra.', 'subject_country', validate_to_none), + ('extra.', 'subject_state_or_province_name', validate_to_none), + ('extra.', 'subject_locality_name', validate_to_none), + ('extra.', 'subject_street_address', validate_to_none), + ('extra.', 'subject_postal_code', validate_to_none), + ('extra.', 'subject_surname', validate_to_none), + ('extra.', 'subject_given_name', validate_to_none), + ('extra.', 'subject_email_address', validate_to_none), + ('extra.', 'subject_business_category', validate_to_none), + ('extra.', 'subject_serial_number', validate_to_none), + ('extra.', 'issuer_organization_name', validate_to_none), + ('extra.', 'issuer_organization_unit_name', validate_to_none), + ('extra.', 'issuer_country', validate_to_none), + ('extra.', 'issuer_state_or_province_name', validate_to_none), + ('extra.', 'issuer_locality_name', validate_to_none), + ('extra.', 'issuer_street_address', validate_to_none), + ('extra.', 'issuer_postal_code', validate_to_none), + ('extra.', 'issuer_surname', validate_to_none), + ('extra.', 'issuer_given_name', validate_to_none), + ('extra.', 'issuer_email_address', validate_to_none), + ('extra.', 'issuer_business_category', validate_to_none), + ('extra.', 'issuer_serial_number', validate_to_none), + ('extra.', 'sha256_fingerprint', validate_to_none), + ('extra.', 'sha512_fingerprint', validate_to_none), + ('extra.', 'md5_fingerprint', validate_to_none), + ('extra.', 'cert_valid', convert_bool), + ('extra.', 'self_signed', convert_bool), + ('extra.', 'cert_expired', convert_bool), + ('extra.', 'validation_level', validate_to_none), + ('extra.', 'browser_trusted', validate_to_none), + ('extra.', 'browser_error', validate_to_none), + ('extra.', 'raw_cert', validate_to_none), + ('extra.', 'raw_cert_chain', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-rsync', - 'protocol.application': 'rsync', + 'protocol.application': 'kubernetes', + 'classification.identifier': 'open-kubernetes', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Accessible-AFP -accessible_afp = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-tcp-report/ +scan_ldap_tcp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('source.local_hostname', 'dns_host_name', validate_to_none), + ('extra.', 'domain_controller_functionality', convert_int), + ('extra.', 'domain_functionality', convert_int), + ('extra.', 'forest_functionality', convert_int), + ('extra.', 'highest_committed_usn', convert_int), + ('extra.', 'is_global_catalog_ready', convert_bool), + ('extra.', 'is_synchronized', convert_bool), ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'accessible-afp' in constant_fields + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_type', validate_to_none), - ('extra.', 'afp_versions', validate_to_none), - ('extra.', 'uams', validate_to_none), - ('extra.', 'flags', validate_to_none), + ('extra.', 'size', convert_int), + ('extra.', 'configuration_naming_context', validate_to_none), + ('extra.', 'current_time', validate_to_none), + ('extra.', 'default_naming_context', validate_to_none), + ('extra.', 'ds_service_name', validate_to_none), + ('extra.', 'ldap_service_name', validate_to_none), + ('extra.', 'naming_contexts', validate_to_none), + ('extra.', 'root_domain_naming_context', validate_to_none), + ('extra.', 'schema_naming_context', validate_to_none), ('extra.', 'server_name', validate_to_none), - ('extra.', 'signature', validate_to_none), - ('extra.', 'directory_service', validate_to_none), - ('extra.', 'utf8_servername', validate_to_none), - ('extra.', 'network_address', validate_to_none), + ('extra.', 'subschema_subentry', validate_to_none), + ('extra.', 'supported_capabilities', validate_to_none), + ('extra.', 'supported_control', validate_to_none), + ('extra.', 'supported_ldap_policies', validate_to_none), + ('extra.', 'supported_ldap_version', validate_to_none), + ('extra.', 'supported_sasl_mechanisms', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'open-ldap', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-afp', - 'protocol.application': 'afp', + 'protocol.application': 'ldap', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Darknet -# legacy (replaced by event4_honeypot_darknet) -darknet = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-tcp-report/ +scan_ldap_udp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('source.port', 'port'), + ('source.local_hostname', 'dns_host_name', validate_to_none), + ('extra.', 'domain_controller_functionality', convert_int), + ('extra.', 'domain_functionality', convert_int), + ('extra.', 'forest_functionality', convert_int), + ('extra.', 'highest_committed_usn', convert_int), + ('extra.', 'is_global_catalog_ready', convert_bool), + ('extra.', 'is_synchronized', convert_bool), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'type', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('extra.', 'count', convert_int), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sic', 'dst_sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'family', validate_to_none), - ('classification.identifier', 'tag'), # different values possible in this report - ('extra.', 'public_source', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.', 'size', convert_int), + ('extra.', 'configuration_naming_context', validate_to_none), + ('extra.', 'current_time', validate_to_none), + ('extra.', 'default_naming_context', validate_to_none), + ('extra.', 'ds_service_name', validate_to_none), + ('extra.', 'ldap_service_name', validate_to_none), + ('extra.', 'naming_contexts', validate_to_none), + ('extra.', 'root_domain_naming_context', validate_to_none), + ('extra.', 'schema_naming_context', validate_to_none), + ('extra.', 'server_name', validate_to_none), + ('extra.', 'subschema_subentry', validate_to_none), + ('extra.', 'supported_capabilities', validate_to_none), + ('extra.', 'supported_control', validate_to_none), + ('extra.', 'supported_ldap_policies', validate_to_none), + ('extra.', 'supported_ldap_version', validate_to_none), + ('extra.', 'supported_sasl_mechanisms', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', + 'classification.identifier': 'open-ldap', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'ldap', }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Amplification-DDoS-Victim -# legacy (replaced by honeypot-ddos-amp) -amplification_ddos_victim = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-mdns-report/ +scan_mdns = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('source.port', 'src_port'), ('protocol.transport', 'protocol'), - ('destination.port', 'dst_port'), ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'tag', validate_to_none), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'request', validate_to_none), - ('extra.', 'count', convert_int), - ('extra.', 'bytes', convert_int), - ('extra.', 'sensor_geo', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'public_source', validate_to_none), + ('extra.', 'mdns_name', validate_to_none), + ('extra.', 'mdns_ipv4', validate_to_none), + ('extra.', 'mdns_ipv6', validate_to_none), + ('extra.', 'services', validate_to_none), + ('extra.', 'workstation_name', validate_to_none), + ('extra.', 'workstation_ipv4', validate_to_none), + ('extra.', 'workstation_ipv6', validate_to_none), + ('extra.', 'workstation_info', validate_to_none), + ('extra.', 'http_name', validate_to_none), + ('extra.', 'http_ipv4', validate_to_none), + ('extra.', 'http_ipv6', validate_to_none), + ('extra.', 'http_ptr', validate_to_none), + ('extra.', 'http_info', validate_to_none), + ('extra.', 'http_target', validate_to_none), + ('extra.', 'http_port', convert_int), + ('extra.', 'spotify_name', validate_to_none), + ('extra.', 'spotify_ipv4', validate_to_none), + ('extra.', 'spotify_ipv6', validate_to_none), + ('extra.', 'opc_ua_discovery', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'amplification-ddos-victim', - } + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'mdns', + 'classification.identifier': 'open-mdns', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/HTTP-Scanners -http_scanners = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-memcached-report/ +scan_memcached = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('extra.', 'pid', convert_int), + ('extra.', 'pointer_size', convert_int), + ('extra.', 'uptime', convert_int), + ('extra.', 'curr_connections', convert_int), + ('extra.', 'total_connections', convert_int), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('destination.ip', 'dst_ip'), - ('destination.port', 'dst_port'), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.fqdn', 'dst_dns', validate_fqdn), - ('extra.', 'type', validate_to_none), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), + ('extra.', 'time', validate_to_none), ('extra.', 'sector', validate_to_none), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'sensorid', validate_to_none), - ('extra.', 'pattern', validate_to_none), - ('extra.', 'url', validate_to_none), - ('extra.file.md5', 'file_md5', validate_to_none), - ('extra.file.sha256', 'file_sha256', validate_to_none), - ('extra.', 'request_raw', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'classification.identifier': 'http', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - } + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'memcached', + 'classification.identifier': 'open-memcached', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/ICS-Scanners -ics_scanners = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-mongodb-report/ +scan_mongodb = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('source.asn', 'asn', invalidate_zero), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), + ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('protocol.application', 'protocol'), - ('destination.ip', 'dst_ip'), - ('destination.port', 'dst_port'), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.fqdn', 'dst_dns', validate_fqdn), - ('extra.', 'type', validate_to_none), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), + ('extra.', 'gitversion', validate_to_none), + ('extra.', 'sysinfo', validate_to_none), + ('extra.', 'opensslversion', validate_to_none), + ('extra.', 'allocator', validate_to_none), + ('extra.', 'javascriptengine', validate_to_none), + ('extra.', 'bits', validate_to_none), + ('extra.', 'maxbsonobjectsize', validate_to_none), + ('extra.', 'ok', validate_to_none), + ('extra.', 'visible_databases', validate_to_none), ('extra.', 'sector', validate_to_none), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'sensorid', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'slave_id', validate_to_none), - ('extra.', 'function_code', convert_int), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', convert_int), ], 'constant_fields': { - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'classification.identifier': 'ics', - } + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'mongodb', + 'classification.identifier': 'open-mongodb', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Ubiquiti -accessible_ubiquiti_discovery_service = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ +scan_mqtt = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('extra.', 'anonymous_access', convert_bool), ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'accessible-ubiquiti-discovery-service' in constant_fields + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.mac_address', 'mac', validate_to_none), - ('extra.radio_name', 'radioname', validate_to_none), - ('extra.', 'essid', validate_to_none), - ('extra.model', 'modelshort', validate_to_none), - ('extra.model_full', 'modelfull', validate_to_none), - ('extra.firmwarerev', 'firmware', validate_to_none), - ('extra.response_size', 'size', convert_int), + ('extra.', 'raw_response', validate_to_none), + ('extra.', 'hex_code', validate_to_none), + ('extra.', 'code', validate_to_none), + ('extra.', 'cipher_suite', validate_to_none), + ('extra.', 'cert_length', convert_int), + ('extra.', 'subject_common_name', validate_to_none), + ('extra.', 'issuer_common_name', validate_to_none), + ('extra.', 'cert_issue_date', validate_to_none), + ('extra.', 'cert_expiration_date', validate_to_none), + ('extra.', 'sha1_fingerprint', validate_to_none), + ('extra.', 'sha256_fingerprint', validate_to_none), + ('extra.', 'sha512_fingerprint', validate_to_none), + ('extra.', 'md5_fingerprint', validate_to_none), + ('extra.', 'cert_serial_number', validate_to_none), + ('extra.', 'ssl_version', convert_int), + ('extra.', 'signature_algorithm', validate_to_none), + ('extra.', 'key_algorithm', validate_to_none), + ('extra.', 'subject_organization_name', validate_to_none), + ('extra.', 'subject_organization_unit_name', validate_to_none), + ('extra.', 'subject_country', validate_to_none), + ('extra.', 'subject_state_or_province_name', validate_to_none), + ('extra.', 'subject_locality_name', validate_to_none), + ('extra.', 'subject_street_address', validate_to_none), + ('extra.', 'subject_postal_code', validate_to_none), + ('extra.', 'subject_surname', validate_to_none), + ('extra.', 'subject_given_name', validate_to_none), + ('extra.', 'subject_email_address', validate_to_none), + ('extra.', 'subject_business_category', validate_to_none), + ('extra.', 'subject_serial_number', validate_to_none), + ('extra.', 'issuer_organization_name', validate_to_none), + ('extra.', 'issuer_organization_unit_name', validate_to_none), + ('extra.', 'issuer_country', validate_to_none), + ('extra.', 'issuer_state_or_province_name', validate_to_none), + ('extra.', 'issuer_locality_name', validate_to_none), + ('extra.', 'issuer_street_address', validate_to_none), + ('extra.', 'issuer_postal_code', validate_to_none), + ('extra.', 'issuer_surname', validate_to_none), + ('extra.', 'issuer_given_name', validate_to_none), + ('extra.', 'issuer_email_address', validate_to_none), + ('extra.', 'issuer_business_category', validate_to_none), + ('extra.', 'issuer_serialNumber', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - } + 'protocol.application': 'mqtt', + 'classification.identifier': 'open-mqtt', + }, } -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Accessible-FTP -accessible_ftp = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ +scan_mqtt_anon = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'accessible-ftp' in constant_fields + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ('extra.', 'handshake', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.', 'raw_response', validate_to_none), + ('extra.', 'hex_code', validate_to_none), + ('extra.', 'code', validate_to_none), ('extra.', 'cipher_suite', validate_to_none), ('extra.', 'cert_length', convert_int), ('extra.', 'subject_common_name', validate_to_none), @@ -2167,8 +2422,11 @@ def force_base64(value: Optional[str]) -> Optional[str]: ('extra.', 'cert_issue_date', validate_to_none), ('extra.', 'cert_expiration_date', validate_to_none), ('extra.', 'sha1_fingerprint', validate_to_none), + ('extra.', 'sha256_fingerprint', validate_to_none), + ('extra.', 'sha512_fingerprint', validate_to_none), + ('extra.', 'md5_fingerprint', validate_to_none), ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', validate_to_none), + ('extra.', 'ssl_version', convert_int), ('extra.', 'signature_algorithm', validate_to_none), ('extra.', 'key_algorithm', validate_to_none), ('extra.', 'subject_organization_name', validate_to_none), @@ -2194,872 +2452,1591 @@ def force_base64(value: Optional[str]) -> Optional[str]: ('extra.', 'issuer_given_name', validate_to_none), ('extra.', 'issuer_email_address', validate_to_none), ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'auth_tls_response', validate_to_none), - ('extra.', 'auth_ssl_response', validate_to_none) + ('extra.', 'issuer_serialNumber', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'protocol.application': 'ftp', - } + 'protocol.application': 'mqtt', + 'classification.identifier': 'open-mqtt-anon', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -open_mqtt = { +# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-MSSQL +scan_mssql = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('source.local_hostname', 'server_name', validate_to_none), + ('extra.', 'tcp_port', convert_int), ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-mqtt' in constant_fields + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'anonymous_access', convert_bool), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none) + ('extra.', 'instance_name', validate_to_none), + ('extra.', 'named_pipe', validate_to_none), + ('extra.', 'response_length', convert_int), + ('extra.', 'amplification', convert_float), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-mqtt', - 'protocol.application': 'mqtt', - } + 'protocol.application': 'mssql', + 'classification.identifier': 'open-mssql', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipp-report/ -open_ipp = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-mysql-server-report/ +scan_mysql = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('extra.', 'client_can_handle_expired_passwords', convert_bool), + ('extra.', 'client_compress', convert_bool), + ('extra.', 'client_connect_attrs', convert_bool), + ('extra.', 'client_connect_with_db', convert_bool), + ('extra.', 'client_deprecated_eof', convert_bool), + ('extra.', 'client_found_rows', convert_bool), + ('extra.', 'client_ignore_sigpipe', convert_bool), + ('extra.', 'client_ignore_space', convert_bool), + ('extra.', 'client_interactive', convert_bool), + ('extra.', 'client_local_files', convert_bool), + ('extra.', 'client_long_flag', convert_bool), + ('extra.', 'client_long_password', convert_bool), + ('extra.', 'client_multi_results', convert_bool), + ('extra.', 'client_multi_statements', convert_bool), + ('extra.', 'client_no_schema', convert_bool), + ('extra.', 'client_odbc', convert_bool), + ('extra.', 'client_plugin_auth', convert_bool), + ('extra.', 'client_plugin_auth_len_enc_client_data', convert_bool), + ('extra.', 'client_protocol_41', convert_bool), + ('extra.', 'client_ps_multi_results', convert_bool), + ('extra.', 'client_reserved', convert_bool), + ('extra.', 'client_secure_connection', convert_bool), + ('extra.', 'client_session_track', convert_bool), + ('extra.', 'client_ssl', convert_bool), + ('extra.', 'client_transactions', convert_bool), ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'open-ipp' in constant_fields + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ipp_version', validate_to_none), - ('extra.', 'cups_version', validate_to_none), - ('extra.', 'printer_uris', validate_to_none), - ('extra.', 'printer_name', validate_to_none), - ('extra.', 'printer_info', validate_to_none), - ('extra.', 'printer_more_info', validate_to_none), - ('extra.', 'printer_make_and_model', validate_to_none), - ('extra.', 'printer_firmware_name', validate_to_none), - ('extra.', 'printer_firmware_string_version', validate_to_none), - ('extra.', 'printer_firmware_version', validate_to_none), - ('extra.', 'printer_organization', validate_to_none), - ('extra.', 'printer_organization_unit', validate_to_none), - ('extra.', 'printer_uuid', validate_to_none), - ('extra.', 'printer_wifi_ssid', validate_to_none) + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'mysql_protocol_version', validate_to_none), + ('extra.', 'server_version', validate_to_none), + ('extra.', 'error_code', validate_to_none), + ('extra.', 'error_id', validate_to_none), + ('extra.', 'error_message', validate_to_none), + ('extra.', 'handshake', validate_to_none), + ('extra.', 'cipher_suite', validate_to_none), + ('extra.', 'cert_length', convert_int), + ('extra.', 'subject_common_name', validate_to_none), + ('extra.', 'issuer_common_name', validate_to_none), + ('extra.', 'cert_issue_date', validate_to_none), + ('extra.', 'cert_expiration_date', validate_to_none), + ('extra.', 'sha1_fingerprint', validate_to_none), + ('extra.', 'cert_serial_number', validate_to_none), + ('extra.', 'ssl_version', convert_int), + ('extra.', 'signature_algorithm', validate_to_none), + ('extra.', 'key_algorithm', validate_to_none), + ('extra.', 'subject_organization_name', validate_to_none), + ('extra.', 'subject_organization_unit_name', validate_to_none), + ('extra.', 'subject_country', validate_to_none), + ('extra.', 'subject_state_or_province_name', validate_to_none), + ('extra.', 'subject_locality_name', validate_to_none), + ('extra.', 'subject_street_address', validate_to_none), + ('extra.', 'subject_postal_code', validate_to_none), + ('extra.', 'subject_surname', validate_to_none), + ('extra.', 'subject_given_name', validate_to_none), + ('extra.', 'subject_email_address', validate_to_none), + ('extra.', 'subject_business_category', validate_to_none), + ('extra.', 'subject_serial_number', validate_to_none), + ('extra.', 'issuer_organization_name', validate_to_none), + ('extra.', 'issuer_organization_unit_name', validate_to_none), + ('extra.', 'issuer_country', validate_to_none), + ('extra.', 'issuer_state_or_province_name', validate_to_none), + ('extra.', 'issuer_locality_name', validate_to_none), + ('extra.', 'issuer_street_address', validate_to_none), + ('extra.', 'issuer_postal_code', validate_to_none), + ('extra.', 'issuer_surname', validate_to_none), + ('extra.', 'issuer_given_name', validate_to_none), + ('extra.', 'issuer_email_address', validate_to_none), + ('extra.', 'issuer_business_category', validate_to_none), + ('extra.', 'issuer_serial_number', validate_to_none), + ('extra.', 'sha256_fingerprint', validate_to_none), + ('extra.', 'sha512_fingerprint', validate_to_none), + ('extra.', 'md5_fingerprint', validate_to_none), + ('extra.', 'cert_valid', convert_bool), + ('extra.', 'self_signed', convert_bool), + ('extra.', 'cert_expired', convert_bool), + ('extra.', 'validation_level', validate_to_none), + ('extra.', 'browser_trusted', validate_to_none), + ('extra.', 'browser_error', validate_to_none), + ('extra.', 'raw_cert', validate_to_none), + ('extra.', 'raw_cert_chain', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ipp', - 'protocol.application': 'ipp', - } + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'protocol.application': 'mysql', + 'classification.identifier': 'open-mysql', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-coap-report/ -accessible_coap = { +# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NATPMP +scan_nat_pmp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - # ('classification.identifier', 'tag'), # always set to 'accessible-coap' in constant_fields + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'response', validate_to_none) + ('extra.', 'opcode', validate_to_none), + ('extra.', 'uptime', validate_to_none), + ('extra.', 'external_ip', validate_to_none), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'open-natpmp', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-coap', - 'protocol.application': 'coap', - } + 'protocol.application': 'natpmp', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-apple-remote-desktop-ard-report/ -accessible_ard = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-netbios-report/ +scan_netbios = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('source.account', 'username'), + ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'mac_address', validate_to_none), ('source.asn', 'asn', invalidate_zero), - # ('classification.identifier', 'tag'), # always 'ard' - set in constant fields ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('protocol.transport', 'protocol'), + ('extra.', 'workgroup', validate_to_none), + ('extra.', 'machine_name', validate_to_none), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'response_size', convert_int), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { + 'classification.identifier': 'open-netbios-nameservice', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ard', - } + 'protocol.application': 'netbios-nameservice', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-radmin-report/ -accessible_radmin = { +# https://www.shadowserver.org/what-we-do/network-reporting/netcore-netis-router-vulnerability-scan-report/ +scan_netis_router = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), + ('source.ip', 'ip', validate_ip), ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('source.asn', 'asn', convert_int), - # ('classification.identifier', 'tag'), # always 'accessible-radmin' - set in constant_fields + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'response', validate_to_none), + ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'naics', convert_int), - ('extra.', 'version', validate_to_none), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { - 'classification.identifier': 'accessible-radmin', + 'classification.identifier': 'open-netis', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - } + 'protocol.transport': 'udp', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/caida-ip-spoofer-report/ -# NOTE: The "type" field is included twice with the same values -# legacy (replaced by event4_ip_spoofer) -caida = { +# https://www.shadowserver.org/what-we-do/network-reporting/ntp-version-report/ +scan_ntp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('source.asn', 'asn', convert_int), - # ('classification.identifier', 'tag'), # always 'ip-spoofer' - set in constant_fields - ('classification.identifier', 'infection'), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname', validate_to_none), - ('extra.', 'type', validate_to_none), - ('extra.', 'naics', convert_int), - ('extra.', 'sic', convert_int), - ('extra.', 'sector', validate_to_none), - # FIXME Is is mappable to some classification.* field? Not included in example data. - ('extra.', 'family', validate_to_none), - ('source.network', 'network', validate_to_none), - (False, 'version', validate_to_none), # we can ignore the IP version, it's obvious from the address - ('extra.', 'routedspoof', validate_to_none), - ('extra.', 'session', convert_int), - ('extra.', 'nat', convert_bool), - ('extra.', 'public_source', validate_to_none), + ('extra.', 'clk_wander', convert_float), + ('extra.', 'frequency', convert_float), + ('extra.', 'jitter', convert_float), + ('extra.', 'leap', convert_float), + ('extra.', 'offset', convert_float), + ('extra.', 'peer', convert_int), + ('extra.', 'poll', convert_int), + ('extra.', 'precision', convert_int), + ('extra.', 'rootdelay', convert_float), + ('extra.', 'rootdispersion', convert_float), + ('extra.', 'stratum', convert_int), + ('extra.', 'tc', convert_int), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'version', validate_to_none), + ('extra.', 'clock', validate_to_none), + ('extra.', 'error', validate_to_none), + ('extra.', 'mintc', validate_to_none), + ('extra.', 'noise', validate_to_none), + ('extra.', 'phase', validate_to_none), + ('extra.', 'processor', validate_to_none), + ('extra.', 'refid', validate_to_none), + ('extra.', 'reftime', validate_to_none), + ('extra.', 'stability', validate_to_none), + ('extra.', 'state', validate_to_none), + ('extra.', 'system', validate_to_none), + ('extra.', 'tai', validate_to_none), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { - # FIXME Check if the classification is correct - 'classification.identifier': 'ip-spoofer', - 'classification.taxonomy': 'fraud', - 'classification.type': 'masquerade', - } + 'classification.identifier': 'ntp-version', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'ntp', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ms-rdpeudp/ -accessible_msrdpeudp = { +# https://www.shadowserver.org/what-we-do/network-reporting/ntp-monitor-report/ +scan_ntpmonitor = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int) + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), - ('source.asn', 'asn', convert_int), + ('source.reverse_dns', 'hostname'), + ('extra.', 'packets', convert_int), + ('extra.', 'size', convert_int), + ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag'), - ('extra.', 'naics', convert_int), - ('extra.', 'sic', convert_int), - ('extra.', 'sessionid'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { - 'classification.identifier': 'accessible-msrdpeudp', + 'classification.identifier': 'ntp-monitor', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - } + 'protocol.application': 'ntp', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-dns-report/ -sinkhole_dns = { +# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Portmapper +scan_portmapper = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int) + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('source.reverse_dns', 'host'), - ('source.asn', 'asn', convert_int), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), - ('extra.dns_query_type', 'type'), - ('extra.dns_query', 'query'), - ('extra.', 'count', convert_int), - ('extra.', 'response'), - ('extra.', 'tag'), - ('extra.', 'sector'), - ('extra.', 'naics', convert_int), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'programs', validate_to_none), + ('extra.', 'mountd_port', validate_to_none), + ('extra.', 'exports', validate_to_none), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { - 'classification.identifier': 'sinkholedns', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - } + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'portmapper', + 'classification.identifier': 'open-portmapper', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-amplification-ddos-events-report/ -honeypot_ddos_amp = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-postgresql-server-report/ +scan_postgres = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('extra.', 'startup_error_line', convert_int), + ('extra.', 'client_ssl', convert_bool), ('protocol.transport', 'protocol'), - ('source.port', 'src_port', convert_int), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname'), - ('malware.name', 'infection'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', invalidate_zero), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'family', validate_to_none), + ('source.reverse_dns', 'hostname'), ('extra.', 'tag', validate_to_none), - ('extra.', 'application', validate_to_none), ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'count', convert_int), - ('extra.', 'bytes', convert_int), - ('extra.', 'end_time', convert_date_utc), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'supported_protocols', validate_to_none), + ('extra.', 'protocol_error_code', validate_to_none), + ('extra.', 'protocol_error_file', validate_to_none), + ('extra.', 'protocol_error_line', validate_to_none), + ('extra.', 'protocol_error_message', validate_to_none), + ('extra.', 'protocol_error_routine', validate_to_none), + ('extra.', 'protocol_error_severity', validate_to_none), + ('extra.', 'protocol_error_severity_v', validate_to_none), + ('extra.', 'startup_error_code', validate_to_none), + ('extra.', 'startup_error_file', validate_to_none), + ('extra.', 'startup_error_message', validate_to_none), + ('extra.', 'startup_error_routine', validate_to_none), + ('extra.', 'startup_error_severity', validate_to_none), + ('extra.', 'startup_error_severity_v', validate_to_none), + ('extra.', 'handshake', validate_to_none), + ('extra.', 'cipher_suite', validate_to_none), + ('extra.', 'cert_length', convert_int), + ('extra.', 'subject_common_name', validate_to_none), + ('extra.', 'issuer_common_name', validate_to_none), + ('extra.', 'cert_issue_date', validate_to_none), + ('extra.', 'cert_expiration_date', validate_to_none), + ('extra.', 'sha1_fingerprint', validate_to_none), + ('extra.', 'cert_serial_number', validate_to_none), + ('extra.', 'ssl_version', convert_int), + ('extra.', 'signature_algorithm', validate_to_none), + ('extra.', 'key_algorithm', validate_to_none), + ('extra.', 'subject_organization_name', validate_to_none), + ('extra.', 'subject_organization_unit_name', validate_to_none), + ('extra.', 'subject_country', validate_to_none), + ('extra.', 'subject_state_or_province_name', validate_to_none), + ('extra.', 'subject_locality_name', validate_to_none), + ('extra.', 'subject_street_address', validate_to_none), + ('extra.', 'subject_postal_code', validate_to_none), + ('extra.', 'subject_surname', validate_to_none), + ('extra.', 'subject_given_name', validate_to_none), + ('extra.', 'subject_email_address', validate_to_none), + ('extra.', 'subject_business_category', validate_to_none), + ('extra.', 'subject_serial_number', validate_to_none), + ('extra.', 'issuer_organization_name', validate_to_none), + ('extra.', 'issuer_organization_unit_name', validate_to_none), + ('extra.', 'issuer_country', validate_to_none), + ('extra.', 'issuer_state_or_province_name', validate_to_none), + ('extra.', 'issuer_locality_name', validate_to_none), + ('extra.', 'issuer_street_address', validate_to_none), + ('extra.', 'issuer_postal_code', validate_to_none), + ('extra.', 'issuer_surname', validate_to_none), + ('extra.', 'issuer_given_name', validate_to_none), + ('extra.', 'issuer_email_address', validate_to_none), + ('extra.', 'issuer_business_category', validate_to_none), + ('extra.', 'issuer_serial_number', validate_to_none), + ('extra.', 'sha256_fingerprint', validate_to_none), + ('extra.', 'sha512_fingerprint', validate_to_none), + ('extra.', 'md5_fingerprint', validate_to_none), + ('extra.', 'cert_valid', convert_bool), + ('extra.', 'self_signed', convert_bool), + ('extra.', 'cert_expired', convert_bool), + ('extra.', 'validation_level', validate_to_none), + ('extra.', 'browser_trusted', validate_to_none), + ('extra.', 'browser_error', validate_to_none), + ('extra.', 'raw_cert', validate_to_none), + ('extra.', 'raw_cert_chain', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'amplification-ddos-victim', - } + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'protocol.application': 'postgres', + 'classification.identifier': 'open-postgres', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-brute-force-events-report/ -honeypot_brute_force = { +# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-QOTD +scan_qotd = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip'), - ('source.port', 'src_port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port'), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname'), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), + ('source.reverse_dns', 'hostname'), ('extra.', 'tag', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('classification.identifier', 'service'), - ('extra.', 'start_time', validate_to_none), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'client_version', validate_to_none), - ('destination.account', 'username', validate_to_none), - ('extra.', 'password', validate_to_none), - ('extra.', 'payload_url', validate_to_none), - ('extra.', 'payload_md5', validate_to_none), + ('extra.', 'quote', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - } + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'qotd', + 'classification.identifier': 'open-qotd', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/ip-spoofer-events-report/ -event4_ip_spoofer = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-quic-report/ +scan_quic = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip'), - ('source.port', 'src_port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', convert_int), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('classification.identifier', 'infection'), - ('extra.', 'family', validate_to_none), + ('source.reverse_dns', 'hostname'), ('extra.', 'tag', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('source.network', 'network', validate_to_none), - ('extra.', 'routedspoof', validate_to_none), - ('extra.', 'session', convert_int), - ('extra.', 'nat', convert_bool), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.', 'version_field_1', validate_to_none), + ('extra.', 'version_field_2', validate_to_none), + ('extra.', 'version_field_3', validate_to_none), + ('extra.', 'version_field_4', validate_to_none), ], 'constant_fields': { - # FIXME Check if the classification is correct - 'classification.identifier': 'ip-spoofer', - 'classification.taxonomy': 'fraud', - 'classification.type': 'masquerade', - } + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'classification.identifier': 'open-quic', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-darknet-events-report/ -event4_honeypot_darknet = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-radmin-report/ +scan_radmin = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('source.port', 'src_port'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', convert_int), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ], + 'constant_fields': { + 'classification.identifier': 'accessible-radmin', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rdp-report/ +scan_rdp = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('extra.', 'cve20190708_vulnerable', convert_bool), + ('extra.', 'bluekeep_vulnerable', convert_bool), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'handshake', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'rdp_protocol', validate_to_none), + ('extra.', 'cert_length', convert_int), + ('extra.', 'subject_common_name', validate_to_none), + ('extra.', 'issuer_common_name', validate_to_none), + ('extra.', 'cert_issue_date', validate_to_none), + ('extra.', 'cert_expiration_date', validate_to_none), + ('extra.', 'sha1_fingerprint', validate_to_none), + ('extra.', 'cert_serial_number', validate_to_none), + ('extra.', 'ssl_version', convert_int), + ('extra.', 'signature_algorithm', validate_to_none), + ('extra.', 'key_algorithm', validate_to_none), + ('extra.', 'sha256_fingerprint', validate_to_none), + ('extra.', 'sha512_fingerprint', validate_to_none), + ('extra.', 'md5_fingerprint', validate_to_none), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'sector', validate_to_none), + ('extra.', 'tlsv13_support', validate_to_none), + ('extra.', 'tlsv13_cipher', validate_to_none), + ('extra.', 'jarm', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'rdp', + 'protocol.transport': 'tcp', + 'classification.identifier': 'open-rdp', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ms-rdpeudp/ +scan_rdpeudp = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'sessionid', validate_to_none), + ], + 'constant_fields': { + 'classification.identifier': 'accessible-msrdpeudp', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + }, +} + +# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Redis +scan_redis = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('extra.os.name', 'os', validate_to_none), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'version', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'git_sha1', validate_to_none), + ('extra.', 'git_dirty_flag', validate_to_none), + ('extra.', 'build_id', validate_to_none), + ('extra.', 'mode', validate_to_none), + ('extra.', 'architecture', validate_to_none), + ('extra.', 'multiplexing_api', validate_to_none), + ('extra.', 'gcc_version', validate_to_none), + ('extra.', 'process_id', validate_to_none), + ('extra.', 'run_id', validate_to_none), + ('extra.', 'uptime', validate_to_none), + ('extra.', 'connected_clients', validate_to_none), + ('extra.', 'sector', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'redis', + 'classification.identifier': 'open-redis', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rsync-report/ +scan_rsync = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('extra.', 'password', convert_bool), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'module', validate_to_none), + ('extra.', 'motd', validate_to_none), + ], + 'constant_fields': { + 'classification.identifier': 'accessible-rsync', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'rsync', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smb-report/ +scan_smb = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('extra.', 'smb_implant', convert_bool), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'arch', validate_to_none), + ('extra.', 'key', validate_to_none), + ('extra.', 'smbv1_support', validate_to_none), + ('extra.', 'smb_major_number', validate_to_none), + ('extra.', 'smb_minor_number', validate_to_none), + ('extra.', 'smb_revision', validate_to_none), + ('extra.', 'smb_version_string', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'classification.identifier': 'open-smb', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smtp-report/ +scan_smtp = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.', 'banner', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'protocol.application': 'smtp', + 'classification.identifier': 'open-smtp', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-smtp-report/ +scan_smtp_vulnerable = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'banner', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'smtp', + 'classification.identifier': 'vulnerable-smtp', + }, +} + +# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SNMP +scan_snmp = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('extra.', 'version', convert_int), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'sysdesc', validate_to_none), + ('extra.', 'sysname', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'sector', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), ('extra.', 'device_type', validate_to_none), ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname'), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('classification.identifier', 'tag'), # different values possible in this report - ('extra.', 'application', validate_to_none), + ('extra.', 'device_version', validate_to_none), + ('extra.', 'device_sector', validate_to_none), + ('extra.', 'tag', validate_to_none), + ('extra.', 'community', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'snmp', + 'classification.identifier': 'open-snmp', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-socks4-5-proxy-report/ +scan_socks = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('protocol.application', 'tag'), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'classification.identifier': 'open-socks', + }, +} + +# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SSDP +scan_ssdp = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'header', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'systime', validate_to_none), + ('extra.', 'cache_control', validate_to_none), + ('extra.', 'location', validate_to_none), + ('extra.', 'server', validate_to_none), + ('extra.', 'search_target', validate_to_none), + ('extra.', 'unique_service_name', validate_to_none), + ('extra.', 'host', validate_to_none), + ('extra.', 'nts', validate_to_none), + ('extra.', 'nt', validate_to_none), + ('extra.', 'content_type', validate_to_none), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'sector', validate_to_none), + ('extra.', 'server_port', validate_to_none), + ('extra.', 'instance', validate_to_none), ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'count', convert_int), + ('extra.', 'updated_at', validate_to_none), + ('extra.', 'resource_identifier', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'ssdp', + 'classification.identifier': 'open-ssdp', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssh-report/ +scan_ssh = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.', 'serverid_raw', validate_to_none), + ('extra.', 'serverid_version', validate_to_none), + ('extra.', 'serverid_software', validate_to_none), + ('extra.', 'serverid_comment', validate_to_none), + ('extra.', 'server_cookie', validate_to_none), + ('extra.', 'available_kex', validate_to_none), + ('extra.', 'available_ciphers', validate_to_none), + ('extra.', 'available_mac', validate_to_none), + ('extra.', 'available_compression', validate_to_none), + ('extra.', 'selected_kex', validate_to_none), + ('extra.', 'algorithm', validate_to_none), + ('extra.', 'selected_cipher', validate_to_none), + ('extra.', 'selected_mac', validate_to_none), + ('extra.', 'selected_compression', validate_to_none), + ('extra.', 'server_signature_value', validate_to_none), + ('extra.', 'server_signature_raw', validate_to_none), + ('extra.', 'server_host_key', validate_to_none), + ('extra.', 'server_host_key_sha256', validate_to_none), + ('extra.', 'rsa_prime', validate_to_none), + ('extra.', 'rsa_prime_length', validate_to_none), + ('extra.', 'rsa_generator', validate_to_none), + ('extra.', 'rsa_generator_length', validate_to_none), + ('extra.', 'rsa_public_key', validate_to_none), + ('extra.', 'rsa_public_key_length', validate_to_none), + ('extra.', 'rsa_exponent', validate_to_none), + ('extra.', 'rsa_modulus', validate_to_none), + ('extra.', 'rsa_length', validate_to_none), + ('extra.', 'dss_prime', validate_to_none), + ('extra.', 'dss_prime_length', validate_to_none), + ('extra.', 'dss_generator', validate_to_none), + ('extra.', 'dss_generator_length', validate_to_none), + ('extra.', 'dss_public_key', validate_to_none), + ('extra.', 'dss_public_key_length', validate_to_none), + ('extra.', 'dss_dsa_public_g', validate_to_none), + ('extra.', 'dss_dsa_public_p', validate_to_none), + ('extra.', 'dss_dsa_public_q', validate_to_none), + ('extra.', 'dss_dsa_public_y', validate_to_none), + ('extra.', 'ecdsa_curve25519', validate_to_none), + ('extra.', 'ecdsa_curve', validate_to_none), + ('extra.', 'ecdsa_public_key_length', validate_to_none), + ('extra.', 'ecdsa_public_key_b', validate_to_none), + ('extra.', 'ecdsa_public_key_gx', validate_to_none), + ('extra.', 'ecdsa_public_key_gy', validate_to_none), + ('extra.', 'ecdsa_public_key_n', validate_to_none), + ('extra.', 'ecdsa_public_key_p', validate_to_none), + ('extra.', 'ecdsa_public_key_x', validate_to_none), + ('extra.', 'ecdsa_public_key_y', validate_to_none), + ('extra.', 'ed25519_curve25519', validate_to_none), + ('extra.', 'ed25519_cert_public_key_nonce', validate_to_none), + ('extra.', 'ed25519_cert_public_key_bytes', validate_to_none), + ('extra.', 'ed25519_cert_public_key_raw', validate_to_none), + ('extra.', 'ed25519_cert_public_key_sha256', validate_to_none), + ('extra.', 'ed25519_cert_public_key_serial', validate_to_none), + ('extra.', 'ed25519_cert_public_key_type_id', validate_to_none), + ('extra.', 'ed25519_cert_public_key_type_name', validate_to_none), + ('extra.', 'ed25519_cert_public_key_keyid', validate_to_none), + ('extra.', 'ed25519_cert_public_key_principles', validate_to_none), + ('extra.', 'ed25519_cert_public_key_valid_after', validate_to_none), + ('extra.', 'ed25519_cert_public_key_valid_before', validate_to_none), + ('extra.', 'ed25519_cert_public_key_duration', validate_to_none), + ('extra.', 'ed25519_cert_public_key_sigkey_bytes', validate_to_none), + ('extra.', 'ed25519_cert_public_key_sigkey_raw', validate_to_none), + ('extra.', 'ed25519_cert_public_key_sigkey_sha256', validate_to_none), + ('extra.', 'ed25519_cert_public_key_sigkey_value', validate_to_none), + ('extra.', 'ed25519_cert_public_key_sig_raw', validate_to_none), + ('extra.', 'banner', validate_to_none), + ('extra.', 'userauth_methods', validate_to_none), + ('extra.', 'device_vendor', validate_to_none), + ('extra.', 'device_type', validate_to_none), + ('extra.', 'device_model', validate_to_none), + ('extra.', 'device_version', validate_to_none), + ('extra.', 'device_sector', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'other', + 'classification.type': 'other', + 'classification.identifier': 'open-ssh', + }, +} + +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssl-report/ +scan_ssl = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'handshake', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'cipher_suite', validate_to_none), + ('extra.', 'ssl_poodle', validate_to_none), + ('extra.', 'cert_length', convert_int), + ('extra.', 'subject_common_name', validate_to_none), + ('extra.', 'issuer_common_name', validate_to_none), + ('extra.', 'cert_issue_date', validate_to_none), + ('extra.', 'cert_expiration_date', validate_to_none), + ('extra.', 'sha1_fingerprint', validate_to_none), + ('extra.', 'cert_serial_number', validate_to_none), + ('extra.', 'ssl_version', convert_int), + ('extra.', 'signature_algorithm', validate_to_none), + ('extra.', 'key_algorithm', validate_to_none), + ('extra.', 'subject_organization_name', validate_to_none), + ('extra.', 'subject_organization_unit_name', validate_to_none), + ('extra.', 'subject_country', validate_to_none), + ('extra.', 'subject_state_or_province_name', validate_to_none), + ('extra.', 'subject_locality_name', validate_to_none), + ('extra.', 'subject_street_address', validate_to_none), + ('extra.', 'subject_postal_code', validate_to_none), + ('extra.', 'subject_surname', validate_to_none), + ('extra.', 'subject_given_name', validate_to_none), + ('extra.', 'subject_email_address', validate_to_none), + ('extra.', 'subject_business_category', validate_to_none), + ('extra.', 'subject_serial_number', validate_to_none), + ('extra.', 'issuer_organization_name', validate_to_none), + ('extra.', 'issuer_organization_unit_name', validate_to_none), + ('extra.', 'issuer_country', validate_to_none), + ('extra.', 'issuer_state_or_province_name', validate_to_none), + ('extra.', 'issuer_locality_name', validate_to_none), + ('extra.', 'issuer_street_address', validate_to_none), + ('extra.', 'issuer_postal_code', validate_to_none), + ('extra.', 'issuer_surname', validate_to_none), + ('extra.', 'issuer_given_name', validate_to_none), + ('extra.', 'issuer_email_address', validate_to_none), + ('extra.', 'issuer_business_category', validate_to_none), + ('extra.', 'issuer_serial_number', validate_to_none), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.', 'freak_vulnerable', validate_to_none), + ('extra.', 'freak_cipher_suite', validate_to_none), + ('extra.source.sector', 'sector', validate_to_none), + ('extra.', 'sha256_fingerprint', validate_to_none), + ('extra.', 'sha512_fingerprint', validate_to_none), + ('extra.', 'md5_fingerprint', validate_to_none), + ('extra.', 'http_response_type', validate_to_none), + ('extra.', 'http_code', convert_int), + ('extra.', 'http_reason', validate_to_none), + ('extra.', 'content_type', validate_to_none), + ('extra.', 'http_connection', validate_to_none), + ('extra.', 'www_authenticate', validate_to_none), + ('extra.', 'set_cookie', validate_to_none), + ('extra.', 'server_type', validate_to_none), + ('extra.', 'content_length', convert_int), + ('extra.', 'transfer_encoding', validate_to_none), + ('extra.', 'http_date', convert_date), + ('extra.', 'cert_valid', convert_bool), + ('extra.', 'self_signed', convert_bool), + ('extra.', 'cert_expired', convert_bool), + ('extra.', 'browser_trusted', validate_to_none), + ('extra.', 'validation_level', validate_to_none), + ('extra.', 'browser_error', validate_to_none), + ('extra.', 'tlsv13_support', validate_to_none), + ('extra.', 'tlsv13_cipher', validate_to_none), + ('extra.', 'jarm', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'other', 'classification.type': 'other', + 'protocol.application': 'https', + 'classification.identifier': 'open-ssl', }, } -# https://www.shadowserver.org/what-we-do/network-reporting/microsoft-sinkhole-events-report/ -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-events-report/ -event46_sinkhole = { +# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Freak-Scan +scan_ssl_freak = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip'), - ('source.port', 'src_port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', convert_int), - ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'freak_vulnerable', convert_bool), + ('extra.', 'browser_trusted', convert_bool), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'handshake', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'cipher_suite', validate_to_none), + ('extra.', 'cert_length', convert_int), + ('extra.', 'subject_common_name', validate_to_none), + ('extra.', 'issuer_common_name', validate_to_none), + ('extra.', 'cert_issue_date', validate_to_none), + ('extra.', 'cert_expiration_date', validate_to_none), + ('extra.', 'sha1_fingerprint', validate_to_none), + ('extra.', 'cert_serial_number', validate_to_none), + ('extra.', 'signature_algorithm', validate_to_none), + ('extra.', 'key_algorithm', validate_to_none), + ('extra.', 'subject_organization_name', validate_to_none), + ('extra.', 'subject_organization_unit_name', validate_to_none), + ('extra.', 'subject_country', validate_to_none), + ('extra.', 'subject_state_or_province_name', validate_to_none), + ('extra.', 'subject_locality_name', validate_to_none), + ('extra.', 'subject_street_address', validate_to_none), + ('extra.', 'subject_postal_code', validate_to_none), + ('extra.', 'subject_surname', validate_to_none), + ('extra.', 'subject_given_name', validate_to_none), + ('extra.', 'subject_email_address', validate_to_none), + ('extra.', 'subject_business_category', validate_to_none), + ('extra.', 'subject_serial_number', validate_to_none), + ('extra.', 'issuer_organization_name', validate_to_none), + ('extra.', 'issuer_organization_unit_name', validate_to_none), + ('extra.', 'issuer_country', validate_to_none), + ('extra.', 'issuer_state_or_province_name', validate_to_none), + ('extra.', 'issuer_locality_name', validate_to_none), + ('extra.', 'issuer_street_address', validate_to_none), + ('extra.', 'issuer_postal_code', validate_to_none), + ('extra.', 'issuer_surname', validate_to_none), + ('extra.', 'issuer_given_name', validate_to_none), + ('extra.', 'issuer_email_address', validate_to_none), + ('extra.', 'issuer_business_category', validate_to_none), + ('extra.', 'issuer_serial_number', validate_to_none), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'freak_cipher_suite', validate_to_none), + ('extra.', 'sector', validate_to_none), + ('extra.', 'sha256_fingerprint', validate_to_none), + ('extra.', 'sha512_fingerprint', validate_to_none), + ('extra.', 'md5_fingerprint', validate_to_none), + ('extra.', 'http_response_type', validate_to_none), + ('extra.', 'http_code', convert_int), + ('extra.', 'http_reason', validate_to_none), + ('extra.', 'content_type', validate_to_none), + ('extra.', 'http_connection', validate_to_none), + ('extra.', 'www_authenticate', validate_to_none), + ('extra.', 'set_cookie', validate_to_none), + ('extra.', 'server_type', validate_to_none), + ('extra.', 'content_length', convert_int), + ('extra.', 'transfer_encoding', validate_to_none), + ('extra.', 'http_date', convert_date), + ('extra.', 'cert_valid', convert_bool), + ('extra.', 'self_signed', convert_bool), + ('extra.', 'cert_expired', convert_bool), + ('extra.', 'validation_level', validate_to_none), + ('extra.', 'browser_error', validate_to_none), + ('extra.', 'tlsv13_support', validate_to_none), + ('extra.', 'tlsv13_cipher', validate_to_none), + ('extra.', 'raw_cert', validate_to_none), + ('extra.', 'raw_cert_chain', validate_to_none), + ('extra.', 'jarm', validate_to_none), ('extra.', 'device_vendor', validate_to_none), ('extra.', 'device_type', validate_to_none), ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port'), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname'), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('classification.identifier', 'tag'), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), + ('extra.', 'device_version', validate_to_none), + ('extra.', 'device_sector', validate_to_none), + ('extra.', 'page_sha256fp', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'https', + 'classification.identifier': 'ssl-freak', }, } -# https://www.shadowserver.org/what-we-do/network-reporting/microsoft-sinkhole-http-events-report/ -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-events-report/ -event46_sinkhole_http = { +# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Scan +scan_ssl_poodle = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip'), - ('source.port', 'src_port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', convert_int), - ('extra.source.sector', 'src_sector', validate_to_none), + ('extra.', 'ssl_poodle', convert_bool), + ('extra.', 'browser_trusted', convert_bool), + ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), + ('extra.', 'handshake', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'cipher_suite', validate_to_none), + ('extra.', 'cert_length', convert_int), + ('extra.', 'subject_common_name', validate_to_none), + ('extra.', 'issuer_common_name', validate_to_none), + ('extra.', 'cert_issue_date', validate_to_none), + ('extra.', 'cert_expiration_date', validate_to_none), + ('extra.', 'sha1_fingerprint', validate_to_none), + ('extra.', 'cert_serial_number', validate_to_none), + ('extra.', 'ssl_version', convert_int), + ('extra.', 'signature_algorithm', validate_to_none), + ('extra.', 'key_algorithm', validate_to_none), + ('extra.', 'subject_organization_name', validate_to_none), + ('extra.', 'subject_organization_unit_name', validate_to_none), + ('extra.', 'subject_country', validate_to_none), + ('extra.', 'subject_state_or_province_name', validate_to_none), + ('extra.', 'subject_locality_name', validate_to_none), + ('extra.', 'subject_street_address', validate_to_none), + ('extra.', 'subject_postal_code', validate_to_none), + ('extra.', 'subject_surname', validate_to_none), + ('extra.', 'subject_given_name', validate_to_none), + ('extra.', 'subject_email_address', validate_to_none), + ('extra.', 'subject_business_category', validate_to_none), + ('extra.', 'subject_serial_number', validate_to_none), + ('extra.', 'issuer_organization_name', validate_to_none), + ('extra.', 'issuer_organization_unit_name', validate_to_none), + ('extra.', 'issuer_country', validate_to_none), + ('extra.', 'issuer_state_or_province_name', validate_to_none), + ('extra.', 'issuer_locality_name', validate_to_none), + ('extra.', 'issuer_street_address', validate_to_none), + ('extra.', 'issuer_postal_code', validate_to_none), + ('extra.', 'issuer_surname', validate_to_none), + ('extra.', 'issuer_given_name', validate_to_none), + ('extra.', 'issuer_email_address', validate_to_none), + ('extra.', 'issuer_business_category', validate_to_none), + ('extra.', 'issuer_serial_number', validate_to_none), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'sector', validate_to_none), + ('extra.', 'sha256_fingerprint', validate_to_none), + ('extra.', 'sha512_fingerprint', validate_to_none), + ('extra.', 'md5_fingerprint', validate_to_none), + ('extra.', 'http_response_type', validate_to_none), + ('extra.', 'http_code', convert_int), + ('extra.', 'http_reason', validate_to_none), + ('extra.', 'content_type', validate_to_none), + ('extra.', 'http_connection', validate_to_none), + ('extra.', 'www_authenticate', validate_to_none), + ('extra.', 'set_cookie', validate_to_none), + ('extra.', 'server_type', validate_to_none), + ('extra.', 'content_length', convert_int), + ('extra.', 'transfer_encoding', validate_to_none), + ('extra.', 'http_date', convert_date), + ('extra.', 'cert_valid', convert_bool), + ('extra.', 'self_signed', convert_bool), + ('extra.', 'cert_expired', convert_bool), + ('extra.', 'validation_level', validate_to_none), + ('extra.', 'browser_error', validate_to_none), + ('extra.', 'tlsv13_support', validate_to_none), + ('extra.', 'tlsv13_cipher', validate_to_none), + ('extra.', 'raw_cert', validate_to_none), + ('extra.', 'raw_cert_chain', validate_to_none), + ('extra.', 'jarm', validate_to_none), ('extra.', 'device_vendor', validate_to_none), ('extra.', 'device_type', validate_to_none), ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port'), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname'), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('classification.identifier', 'tag'), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_agent', validate_to_none), - ('extra.', 'forwarded_by', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), - ('extra.', 'http_referer', validate_to_none), + ('extra.', 'device_version', validate_to_none), + ('extra.', 'device_sector', validate_to_none), + ('extra.', 'page_sha256fp', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'protocol.application': 'http', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'https', + 'classification.identifier': 'ssl-poodle', }, } - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-exchange-server-report/ -def scan_exchange_taxonomy(field): - if field == 'exchange;webshell': - return 'intrusions' - return 'vulnerable' - - -def scan_exchange_type(field): - if field == 'exchange;webshell': - return 'system-compromise' - return 'infected-system' - - -def scan_exchange_identifier(field): - if field == 'exchange;webshell': - return 'exchange-server-webshell' - return 'vulnerable-exchange-server' - - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-exchange-server-report/ -scan_exchange = { +# https://www.shadowserver.org/what-we-do/network-reporting/synful-scan-report/ +scan_synfulknock = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('extra.', 'ack_number', convert_int), + ('extra.', 'window_size', convert_int), + ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - ('extra.', 'tag'), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', convert_int), - ('extra.', 'sic', invalidate_zero), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sic', 'sic', invalidate_zero), + ('extra.', 'sequence_number', validate_to_none), + ('extra.', 'urgent_pointer', validate_to_none), + ('extra.', 'tcp_flags', validate_to_none), + ('extra.', 'raw_packet', validate_to_none), ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('classification.taxonomy', 'tag', scan_exchange_taxonomy), - ('classification.type', 'tag', scan_exchange_type), - ('classification.identifier', 'tag', scan_exchange_identifier), ], 'constant_fields': { + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'classification.identifier': 'open-synfulknock', }, } -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-referer-events-report/ -event46_sinkhole_http_referer = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-telnet-report/ +scan_telnet = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('extra.', 'http_referer_ip', validate_ip), - ('extra.', 'http_referer_asn', convert_int), - ('extra.', 'http_referer_geo', validate_to_none), - ('extra.', 'http_referer_region', validate_to_none), - ('extra.', 'http_referer_city', validate_to_none), - ('extra.', 'http_referer_hostname', validate_to_none), - ('extra.', 'http_referer_naics', invalidate_zero), - ('extra.', 'http_referer_sector', validate_to_none), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname'), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), ('extra.', 'tag', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_referer', validate_to_none), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'banner', validate_to_none), ], 'constant_fields': { - 'classification.identifier': 'sinkhole-http-referer', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - } + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'telnet', + 'classification.identifier': 'open-telnet', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-smtp-report/ -vulnerable_smtp = { +# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-TFTP +scan_tftp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), ('source.ip', 'ip', validate_ip), - ('source.port', 'port'), + ('source.port', 'port', convert_int), ], 'optional_fields': [ + ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), - ('extra.', 'tag'), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), + ('extra.', 'opcode', validate_to_none), + ('extra.', 'errorcode', validate_to_none), + ('extra.', 'error', validate_to_none), + ('extra.', 'errormessage', validate_to_none), + ('extra.', 'size', convert_int), ], 'constant_fields': { - 'classification.identifier': 'vulnerable-smtp', 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'protocol.application': 'smtp', - } + 'protocol.application': 'tftp', + 'classification.identifier': 'open-tftp', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-http-scanner-events/ -honeypot_http_scan = { +# https://www.shadowserver.org/what-we-do/network-reporting/open-ubiquiti-report/ +scan_ubiquiti = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port'), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port'), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname'), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), + ('extra.mac_address', 'mac', validate_to_none), + ('extra.radio_name', 'radioname', validate_to_none), + ('extra.model', 'modelshort', validate_to_none), + ('extra.model_full', 'modelfull', validate_to_none), + ('extra.firmwarerev', 'firmware', validate_to_none), + ('extra.response_size', 'size', convert_int), + ('protocol.transport', 'protocol'), + ('source.reverse_dns', 'hostname'), ('extra.', 'tag', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'pattern', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('user_agent', 'http_agent', validate_to_none), - ('extra.method', 'http_request_method', validate_to_none), - ('extra.', 'url_scheme', validate_to_none), - ('extra.', 'session_tags', validate_to_none), - ('extra.', 'vulnerability_enum', validate_to_none), - ('extra.', 'vulnerability_id', validate_to_none), - ('extra.', 'vulnerability_class', validate_to_none), - ('extra.', 'vulnerability_score', validate_to_none), - ('extra.', 'vulnerability_severity', validate_to_none), - ('extra.', 'vulnerability_version', validate_to_none), - ('extra.', 'threat_framework', validate_to_none), - ('extra.', 'threat_tactic_id', validate_to_none), - ('extra.', 'threat_technique_id', validate_to_none), - ('extra.', 'target_vendor', validate_to_none), - ('extra.', 'target_product', validate_to_none), - ('extra.', 'target_class', validate_to_none), - ('extra.', 'file_md5', validate_to_none), - ('extra.', 'file_sha256', validate_to_none), - ('extra.', 'request_raw', force_base64), - ('extra.', 'body_raw', force_base64), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), + ('extra.', 'sic', invalidate_zero), + ('extra.', 'essid', validate_to_none), ], 'constant_fields': { - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - } + 'classification.identifier': 'accessible-ubiquiti-discovery-service', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ -accessible_amqp = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-vnc-report/ +scan_vnc = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port') + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ - ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'tag'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), - ('extra.', 'channel', convert_int), - ('extra.', 'message_length', convert_int), - ('extra.', 'class', convert_int), - ('extra.', 'method', convert_int), - ('extra.', 'version_major', validate_to_none), - ('extra.', 'version_minor', validate_to_none), - ('extra.', 'capabilities', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'platform', validate_to_none), ('extra.', 'product', validate_to_none), - ('extra.', 'product_version', validate_to_none), - ('extra.', 'mechanisms', validate_to_none), - ('extra.', 'locales', validate_to_none), + ('extra.', 'banner', validate_to_none), + ('extra.', 'sector', validate_to_none), ], 'constant_fields': { 'classification.taxonomy': 'vulnerable', 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-amqp', - } + 'protocol.application': 'vnc', + 'protocol.transport': 'tcp', + 'classification.identifier': 'open-vnc', + }, } -# https://www.shadowserver.org/what-we-do/network-reporting/device-identification-report/ -device_id = { +# https://www.shadowserver.org/what-we-do/network-reporting/accessible-xdmcp-service-report/ +scan_xdmcp = { 'required_fields': [ ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip'), - ('source.port', 'port') + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), ], 'optional_fields': [ ('protocol.transport', 'protocol'), ('source.reverse_dns', 'hostname'), + ('extra.', 'tag', validate_to_none), ('source.asn', 'asn', invalidate_zero), ('source.geolocation.cc', 'geo'), ('source.geolocation.region', 'region'), ('source.geolocation.city', 'city'), - ('extra.', 'tag'), ('extra.', 'naics', invalidate_zero), ('extra.', 'sic', invalidate_zero), + ('extra.', 'opcode', validate_to_none), + ('extra.', 'reported_hostname', validate_to_none), + ('extra.', 'status', validate_to_none), + ('extra.', 'size', convert_int), + ], + 'constant_fields': { + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'protocol.application': 'xdmcp', + 'classification.identifier': 'open-xdmcp', + }, +} + +# http://www.shadowserver.org/wiki/pmwiki.php/Services/Spam-URL +spam_url = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ], + 'optional_fields': [ + ('source.url', 'url', convert_http_host_and_url, True), + ('source.fqdn', 'http_host', validate_fqdn), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('extra.', 'naics', invalidate_zero), ('extra.', 'sector', validate_to_none), + ('extra.', 'source', validate_to_none), + ('extra.', 'sender', validate_to_none), + ('extra.', 'subject', validate_to_none), + ('malware.hash.md5', 'md5', validate_to_none), + ], + 'constant_fields': { + 'classification.taxonomy': 'abusive-content', + 'classification.type': 'spam', + 'classification.identifier': 'spam-url', + }, +} + +special = { + 'required_fields': [ + ('time.source', 'timestamp', add_UTC_to_timestamp), + ('source.ip', 'ip', validate_ip), + ('source.port', 'port', convert_int), + ], + 'optional_fields': [ + ('protocol.transport', 'protocol'), + ('source.asn', 'asn', invalidate_zero), + ('source.geolocation.cc', 'geo'), + ('source.geolocation.region', 'region'), + ('source.geolocation.city', 'city'), + ('source.reverse_dns', 'hostname'), + ('extra.source.naics', 'naics', invalidate_zero), + ('extra.source.sector', 'sector', validate_to_none), + ('malware.name', 'tag'), + ('extra.', 'public_source', validate_to_none), + ('extra.', 'status', validate_to_none), + ('extra.', 'method', validate_to_none), ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), ], 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'undetermined', - 'classification.identifier': 'device-id', - } + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'classification.identifier': 'special', + }, } mapping = ( # feed name, file name, function - ('Accessible-ADB', 'scan_adb', accessible_adb), - ('Accessible-AFP', 'scan_afp', accessible_afp), - ('Accessible-AMQP', 'scan_amqp', accessible_amqp), - ('Accessible-ARD', 'scan_ard', accessible_ard), - ('Accessible-CoAP', 'scan_coap', accessible_coap), - ('Accessible-CWMP', 'scan_cwmp', accessible_cwmp), - ('Accessible-Cisco-Smart-Install', 'cisco_smart_install', accessible_cisco_smart_install), - ('Accessible-FTP', 'scan_ftp', accessible_ftp), - ('Accessible-HTTP', 'scan_http', accessible_vulnerable_http), - ('Accessible-Hadoop', 'scan_hadoop', accessible_hadoop), - ('Accessible-MS-RDPEUDP', 'scan_msrdpeudp', accessible_msrdpeudp), # not used by shadowserver, see line below - ('Accessible-RDPEUDP', 'scan_rdpeudp', accessible_msrdpeudp), - ('Accessible-Radmin', 'scan_radmin', accessible_radmin), - ('Accessible-RDP', 'scan_rdp', accessible_rdp), - ('Accessible-Rsync', 'scan_rsync', accessible_rsync), - ('Accessible-SMB', 'scan_smb', accessible_smb), - ('Accessible-Telnet', 'scan_telnet', accessible_telnet), - ('Accessible-Ubiquiti-Discovery-Service', 'scan_ubiquiti', accessible_ubiquiti_discovery_service), - ('Accessible-VNC', 'scan_vnc', accessible_vnc), - ('Amplification-DDoS-Victim', 'ddos_amplification', amplification_ddos_victim), # legacy (replaced by honeypot-ddos-amp) - ('Blacklisted-IP', 'blacklist', blocklist), ('Blocklist', 'blocklist', blocklist), - ('CAIDA-IP-Spoofer', 'caida_ip_spoofer', caida), # legacy (replaced by event4_ip_spoofer) ('Compromised-Website', 'compromised_website', compromised_website), - ('DNS-Open-Resolvers', 'scan_dns', dns_open_resolvers), - ('Darknet', 'darknet', darknet), # legacy (replaced by event4_honeypot_darknet) ('Device-Identification IPv4', 'device_id', device_id), - ('Device-Identification IPv6', 'device_id6', device_id), - ('Drone', 'botnet_drone', drone), # legacy (replaced by event4_sinkhole, event4_honeypot_darknet and event46_sinkhole_http) - ('Drone-Brute-Force', 'drone_brute_force', drone_brute_force), # legacy (replaced by honeypot_brute_force) - ('HTTP-Scanners', 'hp_http_scan', http_scanners), - ('Honeypot-Amplification-DDoS-Events', 'event4_honeypot_ddos_amp', honeypot_ddos_amp), - ('Honeypot-Brute-Force-Events', 'event4_honeypot_brute_force', honeypot_brute_force), - ('Honeypot-Darknet', 'event4_honeypot_darknet', event4_honeypot_darknet), - ('Honeypot-HTTP-Scan', 'event4_honeypot_http_scan', honeypot_http_scan), - ('ICS-Scanners', 'hp_ics_scan', ics_scanners), - ('IPv6-Sinkhole-HTTP-Drone', 'sinkhole6_http', ipv6_sinkhole_http_drone), # legacy (replaced by event46_sinkhole_http) - ('IP-Spoofer-Events', 'event4_ip_spoofer', event4_ip_spoofer), - ('Microsoft-Sinkhole', 'microsoft_sinkhole', microsoft_sinkhole), # legacy (replaced by event46_sinkhole_http) - ('Microsoft-Sinkhole-Events IPv4', 'event4_microsoft_sinkhole', event46_sinkhole), - ('Microsoft-Sinkhole-Events-HTTP IPv4', 'event4_microsoft_sinkhole_http', event46_sinkhole_http), - ('NTP-Monitor', 'scan_ntpmonitor', ntp_monitor), - ('NTP-Version', 'scan_ntp', ntp_version), - ('Open-Chargen', 'scan_chargen', open_chargen), - ('Open-DB2-Discovery-Service', 'scan_db2', open_db2_discovery_service), - ('Open-Elasticsearch', 'scan_elasticsearch', open_elasticsearch), - ('Open-IPMI', 'scan_ipmi', open_ipmi), - ('Open-IPP', 'scan_ipp', open_ipp), - ('Open-LDAP', 'scan_ldap', open_ldap), - ('Open-LDAP-TCP', 'scan_ldap_tcp', open_ldap), - ('Open-MQTT', 'scan_mqtt', open_mqtt), - ('Open-MSSQL', 'scan_mssql', open_mssql), - ('Open-Memcached', 'scan_memcached', open_memcached), - ('Open-MongoDB', 'scan_mongodb', open_mongodb), - ('Open-NATPMP', 'scan_nat_pmp', open_natpmp), - ('Open-NetBIOS-Nameservice', 'scan_netbios', open_netbios_nameservice), - ('Open-Netis', 'netis_router', open_netis), - ('Open-Portmapper', 'scan_portmapper', open_portmapper), - ('Open-QOTD', 'scan_qotd', open_qotd), - ('Open-Redis', 'scan_redis', open_redis), - ('Open-SNMP', 'scan_snmp', open_snmp), - ('Open-SSDP', 'scan_ssdp', open_ssdp), - ('Open-TFTP', 'scan_tftp', open_tftp), - ('Open-XDMCP', 'scan_xdmcp', open_xdmcp), - ('Open-mDNS', 'scan_mdns', open_mdns), - ('Outdated-DNSSEC-Key', 'outdated_dnssec_key', outdated_dnssec_key), - ('Outdated-DNSSEC-Key-IPv6', 'outdated_dnssec_key_v6', outdated_dnssec_key), - ('SSL-FREAK-Vulnerable-Servers', 'scan_ssl_freak', ssl_freak_vulnerable_servers), - ('SSL-POODLE-Vulnerable-Servers IPv4', 'scan_ssl_poodle', ssl_poodle46_vulnerable_servers), - ('SSL-POODLE-Vulnerable-Servers IPv6', 'scan6_ssl_poodle', ssl_poodle46_vulnerable_servers), - ('Sandbox-URL', 'cwsandbox_url', sandbox_url), - ('Sinkhole-DNS', 'sinkhole_dns', sinkhole_dns), - ('Sinkhole-Events', 'event4_sinkhole', event46_sinkhole), - ('Sinkhole-Events IPv4', 'event4_sinkhole', event46_sinkhole), - ('Sinkhole-Events IPv6', 'event6_sinkhole', event46_sinkhole), - ('Sinkhole-Events-HTTP', 'event4_sinkhole_http', event46_sinkhole_http), - ('Sinkhole-Events-HTTP IPv4', 'event4_sinkhole_http', event46_sinkhole_http), - ('Sinkhole-Events-HTTP IPv6', 'event6_sinkhole_http', event46_sinkhole_http), - ('Sinkhole-HTTP-Drone', 'sinkhole_http_drone', sinkhole_http_drone), # legacy (replaced by event46_sinkhole_http) - ('Sinkhole-Events-HTTP-Referer', 'event4_sinkhole_http_referer', event46_sinkhole_http_referer), - ('Sinkhole-Events-HTTP-Referer IPv4', 'event4_sinkhole_http_referer', event46_sinkhole_http_referer), - ('Sinkhole-Events-HTTP-Referer IPv6', 'event6_sinkhole_http_referer', event46_sinkhole_http_referer), - ('Spam-URL', 'spam_url', spam_url), - ('Vulnerable-ISAKMP', 'scan_isakmp', vulnerable_isakmp), - ('Vulnerable-HTTP', 'scan_http_vulnerable', accessible_vulnerable_http), + ('Device-Identification IPv6', 'device_id6', device_id6), + ('Honeypot-Brute-Force-Events', 'event4_honeypot_brute_force', event_honeypot_brute_force), + ('Honeypot-Darknet', 'event4_honeypot_darknet', event_honeypot_darknet), + ('Honeypot-DDoS', 'event4_honeypot_ddos', event_honeypot_ddos), + ('Honeypot-Amplification-DDoS-Events', 'event4_honeypot_ddos_amp', event_honeypot_ddos_amp), + ('Honeypot-DDoS-Target', 'event4_honeypot_ddos_target', event_honeypot_ddos_target), + ('Honeypot-HTTP-Scan', 'event4_honeypot_http_scan', event_honeypot_http_scan), + ('Honeypot-ICS-Scanner', 'event4_honeypot_ics_scan', event_honeypot_ics_scan), + ('IP-Spoofer-Events', 'event4_ip_spoofer', event_ip_spoofer), + ('Microsoft-Sinkhole-Events IPv4', 'event4_microsoft_sinkhole', event_sinkhole), + ('Microsoft-Sinkhole-Events-HTTP IPv4', 'event4_microsoft_sinkhole_http', event_sinkhole_http), + ('Sinkhole-Events IPv4', 'event4_sinkhole', event_sinkhole), + ('Sinkhole-DNS', 'event4_sinkhole_dns', event_sinkhole_dns), + ('Sinkhole-Events-HTTP IPv4', 'event4_sinkhole_http', event_sinkhole_http), + ('Sinkhole-Events-HTTP-Referer IPv4', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), + ('Sinkhole-Events IPv6', 'event6_sinkhole', event_sinkhole), + ('Sinkhole-Events-HTTP IPv6', 'event6_sinkhole_http', event_sinkhole_http), + ('Sinkhole-Events-HTTP-Referer IPv6', 'event6_sinkhole_http_referer', event_sinkhole_http_referer), + ('Malware-URL', 'malware_url', malware_url), + ('Phish-URL', 'phish_url', phish_url), + ('Sandbox-Connections', 'sandbox_conn', sandbox_conn), + ('Sandbox-DNS', 'sandbox_dns', sandbox_dns), + ('Sandbox-URL', 'sandbox_url', sandbox_url), + ('IPv6-Accessible-CWMP', 'scan6_cwmp', scan_cwmp), + ('IPv6-Accessible-FTP', 'scan6_ftp', scan_ftp), + ('IPv6-Accessible-HTTP', 'scan6_http', scan_http), + ('IPv6-Vulnerable-HTTP', 'scan6_http_vulnerable', scan_http_vulnerable), + ('IPv6-Open-MQTT', 'scan6_mqtt', scan_mqtt), + ('IPv6-Open-Anonymous-MQTT', 'scan6_mqtt_anon', scan_mqtt_anon), + ('IPv6-Accessible-MySQL', 'scan6_mysql', scan_mysql), + ('IPv6-Accessible-PostgreSQL', 'scan6_postgres', scan_postgres), + ('IPv6-Accessible-RDP', 'scan6_rdp', scan_rdp), + ('IPv6-Accessible-SMB', 'scan6_smb', scan_smb), + ('IPv6-Accessible-SMTP', 'scan6_smtp', scan_smtp), + ('IPv6-Vulnerable-SMTP', 'scan6_smtp_vulnerable', scan_smtp_vulnerable), + ('IPv6-Open-SNMP', 'scan6_snmp', scan_snmp), + ('IPv6-Accessible-SSH', 'scan6_ssh', scan_ssh), + ('IPv6-Accessible-SSL', 'scan6_ssl', scan_ssl), + ('SSL-FREAK-Vulnerable-Servers IPv6', 'scan6_ssl_freak', scan_ssl_freak), + ('SSL-POODLE-Vulnerable-Servers IPv6', 'scan6_ssl_poodle', scan_ssl_poodle), + ('IPv6-Accessible-Telnet', 'scan6_telnet', scan_telnet), + ('IPv6-Accessible-VNC', 'scan6_vnc', scan_vnc), + ('Accessible-ADB', 'scan_adb', scan_adb), + ('Accessible-AFP', 'scan_afp', scan_afp), + ('Accessible-AMQP', 'scan_amqp', scan_amqp), + ('Accessible-ARD', 'scan_ard', scan_ard), + ('Open-Chargen', 'scan_chargen', scan_chargen), + ('Accessible-Cisco-Smart-Install', 'scan_cisco_smart_install', scan_cisco_smart_install), + ('Accessible-CoAP', 'scan_coap', scan_coap), + ('Accessible-CouchDB', 'scan_couchdb', scan_couchdb), + ('Accessible-CWMP', 'scan_cwmp', scan_cwmp), + ('Open-DB2-Discovery-Service', 'scan_db2', scan_db2), + ('Vulnerable-DDoS-Middlebox', 'scan_ddos_middlebox', scan_ddos_middlebox), + ('DNS-Open-Resolvers', 'scan_dns', scan_dns), + ('Accessible-Docker', 'scan_docker', scan_docker), + ('Accessible-DVR-DHCPDiscover', 'scan_dvr_dhcpdiscover', scan_dvr_dhcpdiscover), + ('Open-Elasticsearch', 'scan_elasticsearch', scan_elasticsearch), + ('Accessible-Erlang-Port-Mapper-Daemon', 'scan_epmd', scan_epmd), ('Vulnerable-Exchange-Server', 'scan_exchange', scan_exchange), - ('Vulnerable-SMTP', 'scan_smtp_vulnerable', vulnerable_smtp), + ('Accessible-FTP', 'scan_ftp', scan_ftp), + ('Accessible-Hadoop', 'scan_hadoop', scan_hadoop), + ('Accessible-HTTP', 'scan_http', scan_http), + ('Vulnerable-HTTP', 'scan_http_vulnerable', scan_http_vulnerable), + ('Accessible-ICS', 'scan_ics', scan_ics), + ('Open-IPMI', 'scan_ipmi', scan_ipmi), + ('Open-IPP', 'scan_ipp', scan_ipp), + ('Vulnerable-ISAKMP', 'scan_isakmp', scan_isakmp), + ('Accessible-Kubernetes-API', 'scan_kubernetes', scan_kubernetes), + ('Open-LDAP-TCP', 'scan_ldap_tcp', scan_ldap_tcp), + ('Open-LDAP', 'scan_ldap_udp', scan_ldap_udp), + ('Open-mDNS', 'scan_mdns', scan_mdns), + ('Open-Memcached', 'scan_memcached', scan_memcached), + ('Open-MongoDB', 'scan_mongodb', scan_mongodb), + ('Open-MQTT', 'scan_mqtt', scan_mqtt), + ('Open-Anonymous-MQTT', 'scan_mqtt_anon', scan_mqtt_anon), + ('Open-MSSQL', 'scan_mssql', scan_mssql), + ('Accessible-MySQL', 'scan_mysql', scan_mysql), + ('Open-NATPMP', 'scan_nat_pmp', scan_nat_pmp), + ('Open-NetBIOS-Nameservice', 'scan_netbios', scan_netbios), + ('Open-Netis', 'scan_netis_router', scan_netis_router), + ('NTP-Version', 'scan_ntp', scan_ntp), + ('NTP-Monitor', 'scan_ntpmonitor', scan_ntpmonitor), + ('Open-Portmapper', 'scan_portmapper', scan_portmapper), + ('Accessible-PostgreSQL', 'scan_postgres', scan_postgres), + ('Open-QOTD', 'scan_qotd', scan_qotd), + ('Accessible-QUIC', 'scan_quic', scan_quic), + ('Accessible-Radmin', 'scan_radmin', scan_radmin), + ('Accessible-RDP', 'scan_rdp', scan_rdp), + ('Accessible-MS-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), + ('Open-Redis', 'scan_redis', scan_redis), + ('Accessible-Rsync', 'scan_rsync', scan_rsync), + ('Accessible-SMB', 'scan_smb', scan_smb), + ('Accessible-SMTP', 'scan_smtp', scan_smtp), + ('Vulnerable-SMTP', 'scan_smtp_vulnerable', scan_smtp_vulnerable), + ('Open-SNMP', 'scan_snmp', scan_snmp), + ('Accessible-SOCKS4/5-Proxy', 'scan_socks', scan_socks), + ('Open-SSDP', 'scan_ssdp', scan_ssdp), + ('Accessible-SSH', 'scan_ssh', scan_ssh), + ('Accessible-SSL', 'scan_ssl', scan_ssl), + ('SSL-FREAK-Vulnerable-Servers', 'scan_ssl_freak', scan_ssl_freak), + ('SSL-POODLE-Vulnerable-Servers IPv4', 'scan_ssl_poodle', scan_ssl_poodle), + ('SYNful-Knock', 'scan_synfulknock', scan_synfulknock), + ('Accessible-Telnet', 'scan_telnet', scan_telnet), + ('Open-TFTP', 'scan_tftp', scan_tftp), + ('Accessible-Ubiquiti-Discovery-Service', 'scan_ubiquiti', scan_ubiquiti), + ('Accessible-VNC', 'scan_vnc', scan_vnc), + ('Open-XDMCP', 'scan_xdmcp', scan_xdmcp), + ('Spam-URL', 'spam_url', spam_url), + ('Special', 'special', special), + ('Accessible-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), + ('Sinkhole-Events', 'event4_sinkhole', event_sinkhole), + ('Sinkhole-Events-HTTP', 'event4_sinkhole_http', event_sinkhole_http), + ('Sinkhole-Events-HTTP-Referer', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), ) +# END CONFGEN feedname_mapping = {feedname: function for feedname, filename, function in mapping} filename_mapping = {filename: (feedname, function) for feedname, filename, function in mapping} diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 67da8ea29..51fc9b41c 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -85,10 +85,6 @@ def parse_line(self, row, report): conf = self._sparser_config - # https://github.com/certtools/intelmq/issues/1271 - if conf == config.drone and row.get('infection') == 'spam': - conf = config.drone_spam - # we need to copy here... fields = copy.copy(self.csv_fieldnames) # We will use this variable later. diff --git a/intelmq/lib/test.py b/intelmq/lib/test.py index f98007cf1..d7b27eb4c 100644 --- a/intelmq/lib/test.py +++ b/intelmq/lib/test.py @@ -46,7 +46,7 @@ } -class Parameters: +class Parameters(object): pass @@ -63,7 +63,7 @@ def mocked(conf_file): confname = os.path.join('etc/', os.path.split(conf_file)[-1]) fname = pkg_resources.resource_filename('intelmq', confname) - with open(fname) as fpconfig: + with open(fname, 'rt') as fpconfig: return json.load(fpconfig) else: return utils.load_configuration(conf_file) @@ -105,7 +105,7 @@ def skip_build_environment(): return unittest.skipIf(os.getenv('USER') == 'abuild', 'Test disabled in Build Service.') -class BotTestCase: +class BotTestCase(object): """ Provides common tests and assert methods for bot testing. """ @@ -199,11 +199,12 @@ def prepare_bot(self, parameters={}, destination_queues=None, prepare_source_que """ self.log_stream = io.StringIO() - src_name = f"{self.bot_id}-input" + src_name = "{}-input".format(self.bot_id) if not destination_queues: - destination_queues = {"_default": f"{self.bot_id}-output"} + destination_queues = {"_default": "{}-output".format(self.bot_id)} else: - destination_queues = {queue_name: f"{self.bot_id}-{queue_name.strip('_')}-output" + destination_queues = {queue_name: "%s-%s-output" % (self.bot_id, + queue_name.strip('_')) for queue_name in destination_queues} config = BOT_CONFIG.copy() @@ -283,7 +284,7 @@ def test_static_bot_check_method(self, *args, **kwargs): self.assertNotEqual(check[0].upper(), 'ERROR', '%s.check returned the error %r.' '' % (self.bot_name, check[1])) - raise ValueError(f'checks is {checks!r}') + raise ValueError('checks is %r' % (checks, )) def run_bot(self, iterations: int = 1, error_on_pipeline: bool = False, prepare=True, parameters={}, @@ -350,7 +351,7 @@ def run_bot(self, iterations: int = 1, error_on_pipeline: bool = False, """ Test if bot log messages are correctly formatted. """ self.assertLoglineMatches(0, "{} initialized with id {} and intelmq [0-9a-z.]* and python" - r" [0-9a-z.]{{5,8}}\+? \(.+?\)( \[GCC.*?\])?" + r" [0-9a-z.]{{5,8}}\+? \([a-zA-Z0-9,:. ]+\)( \[GCC\])?" r" as process [0-9]+\." "".format(self.bot_name, self.bot_id), "INFO") @@ -424,7 +425,7 @@ def test_bot_name(self, *args, **kwargs): for type_name, type_match in self.bot_types.items(): try: self.assertRegex(self.bot_name, - fr'\A[a-zA-Z0-9]+{type_match}\Z') + r'\A[a-zA-Z0-9]+{}\Z'.format(type_match)) except AssertionError: counter += 1 if counter != len(self.bot_types) - 1: diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index ca250df0c..e9404165c 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -37,6 +37,7 @@ 'v300_pipeline_file_removal', 'v301_deprecations', 'v310_feed_changes', + 'v310_shadowserver_feednames', ] @@ -726,6 +727,36 @@ def v301_deprecations(configuration, harmonization, dry_run, **kwargs): return messages + ' Remove affected bots yourself.' if messages else changed, configuration, harmonization +def v310_shadowserver_feednames(configuration, harmonization, dry_run, **kwargs): + """ + Remove legacy Shadowserver feednames + """ + legacy = { + 'Amplification-DDoS-Victim': 1, + 'Blacklisted-IP': 1, + 'CAIDA-IP-Spoofer': 1, + 'Darknet': 1, + 'Drone': 1, + 'Drone-Brute-Force': 1, + 'HTTP-Scanners': 1, + 'ICS-Scanners': 1, + 'IPv6-Sinkhole-HTTP-Drone': 1, + 'Microsoft-Sinkhole': 1, + 'Outdated-DNSSEC-Key': 1, + 'Outdated-DNSSEC-Key-IPv6': 1, + 'Sinkhole-HTTP-Drone': 1 + } + changed = None + names = [] + for bot_id, bot in configuration.items(): + if bot_id == 'global': + continue + if bot["module"] == "intelmq.bots.parsers.shadowserver.parser": + if bot["parameters"]["feedname"] in legacy: + names.append(bot["parameters"]["feedname"]) + return 'A discontinued feed has been found and must be removed %s' % ', '.join(names) if names else changed, configuration, harmonization + + def v310_feed_changes(configuration, harmonization, dry_run, **kwargs): """ Migrates feeds' configuration for changed/fixed parameter @@ -771,7 +802,7 @@ def v310_feed_changes(configuration, harmonization, dry_run, **kwargs): ((3, 0, 0), (v300_bots_file_removal, v300_defaults_file_removal, v300_pipeline_file_removal, )), ((3, 0, 1), (v301_deprecations, )), ((3, 0, 2), ()), - ((3, 1, 0), (v310_feed_changes, )), + ((3, 1, 0), (v310_feed_changes, v310_shadowserver_feednames, )), ]) ALWAYS = (harmonization, ) diff --git a/intelmq/tests/bots/collectors/shadowserver/README.md b/intelmq/tests/bots/collectors/shadowserver/README.md new file mode 100644 index 000000000..eb0ddfb4a --- /dev/null +++ b/intelmq/tests/bots/collectors/shadowserver/README.md @@ -0,0 +1,9 @@ + + +This module is maintained by [The Shadowserver Foundation](https://www.shadowserver.org/). + +Please contact intelmq@shadowserver.org with any issues or concerns. + diff --git a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py index 0da35b7ef..a625c9d34 100644 --- a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py +++ b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py @@ -13,7 +13,7 @@ RANDSTR = secrets.token_urlsafe(50) ASSET_PATH = pathlib.Path(__file__).parent / 'reports-list.json' -PARAMETERS = {'country': 'anarres', 'api_key': RANDSTR, 'secret': RANDSTR, 'logging_level': 'DEBUG', 'types': ['scan_smb', 'cisco_smart_install', 'nonexistent'], 'name': 'shadowservercollector'} +PARAMETERS = {'reports': 'anarres', 'api_key': RANDSTR, 'secret': RANDSTR, 'logging_level': 'DEBUG', 'types': ['scan_smb', 'cisco_smart_install', 'nonexistent'], 'name': 'shadowservercollector'} REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.json', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} @@ -49,11 +49,11 @@ def test_faulty_config_0(self, mocker): self.assertEqual(str(exception), 'No secret provided.') def test_faulty_config_1(self, mocker): - parameters = {'api_key': RANDSTR, 'secret': RANDSTR} + parameters = {'secret': RANDSTR} with self.assertRaises(ValueError) as context: self.run_bot(iterations=1, parameters=parameters) exception = context.exception - self.assertEqual(str(exception), 'No country provided.') + self.assertEqual(str(exception), 'No api_key provided.') def test_empty_response(self, mocker): mocker.post('https://transform.shadowserver.org/api2/reports/list', text='{}') diff --git a/intelmq/tests/bots/parsers/shadowserver/README.md b/intelmq/tests/bots/parsers/shadowserver/README.md index cf3584454..b96349a70 100644 --- a/intelmq/tests/bots/parsers/shadowserver/README.md +++ b/intelmq/tests/bots/parsers/shadowserver/README.md @@ -1,4 +1,5 @@