Anomaly detection with ML

Makefile

@@ -87,11 +87,9 @@ bool = $(if $(filter $(call lc, $1),1 yes true),1,0)
 
 #--------------------------------------
 #
-# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/
+# Define LD_OPTS for the sub-makefiles in cmd/
 #
 
-MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
-
 LD_OPTS_VARS= \
 -X 'github.com/crowdsecurity/go-cs-lib/version.Version=$(BUILD_VERSION)' \
 -X 'github.com/crowdsecurity/go-cs-lib/version.BuildDate=$(BUILD_TIMESTAMP)' \
@@ -150,6 +148,14 @@ ifneq (,$(RE2_CHECK))
 endif
 endif
 
+#--------------------------------------
+#
+# List of required build-time dependencies
+
+DEPS_DIR := $(CURDIR)/build/deps
+
+DEPS_FILES :=
+
 #--------------------------------------
 #
 # Handle optional components and build profiles, to save space on the final binaries.
@@ -177,6 +183,7 @@ COMPONENTS := \
 	datasource_syslog \
 	datasource_wineventlog \
 	cscli_setup \
+	mlsupport \
 	db_mysql \
 	db_postgres \
 	db_sqlite
@@ -192,6 +199,9 @@ INCLUDE_MINIMAL := db_sqlite datasource_file
 
 EXCLUDE_MINIMAL := $(filter-out $(INCLUDE_MINIMAL),$(strip $(COMPONENTS)))
 
+# ml-support requires pre-built static libraries and weights 20MB
+EXCLUDE_DEFAULT := mlsupport
+
 # example
 # EXCLUDE_MEDIUM := datasource_kafka,datasource_kinesis,datasource_s3
 
@@ -200,8 +210,10 @@ BUILD_PROFILE ?= default
 # Set the EXCLUDE_LIST based on the chosen profile, unless EXCLUDE is already set
 ifeq ($(BUILD_PROFILE),minimal)
 EXCLUDE ?= $(EXCLUDE_MINIMAL)
-else ifneq ($(BUILD_PROFILE),default)
-$(error Invalid build profile specified: $(BUILD_PROFILE). Valid profiles are: minimal, default)
+else ifeq ($(BUILD_PROFILE),default)
+EXCLUDE ?= $(EXCLUDE_DEFAULT)
+else ifneq ($(BUILD_PROFILE),full)
+$(error Invalid build profile specified: $(BUILD_PROFILE). Valid profiles are: minimal, default, full)
 endif
 
 # Create list of excluded components from the EXCLUDE variable
@@ -219,6 +231,24 @@ ifneq ($(COMPONENT_TAGS),)
 GO_TAGS := $(GO_TAGS),$(subst $(space),$(comma),$(COMPONENT_TAGS))
 endif
 
+ifeq ($(filter mlsupport,$(EXCLUDE_LIST)),)
+    $(info mlsupport is included)
+    # Set additional variables when mlsupport is included
+ifneq ($(call bool,$(BUILD_RE2_WASM)),1)
+    $(error for now, the flag BUILD_RE2_WASM is required for mlsupport)
+endif
+    CGO_CPPFLAGS := -I$(DEPS_DIR)/src/onnxruntime/include/onnxruntime/core/session
+    CGO_LDFLAGS := -L$(DEPS_DIR)/libs-lstdc++ -lonnxruntime -dl -lm
+    LIBRARY_PATH := $(DEPS_DIR)/lib
+    DEPS_FILES += $(DEPS_DIR)/lib/libtokenizers.a
+    DEPS_FILES += $(DEPS_DIR)/lib/libonnxruntime.a
+    DEPS_FILES += $(DEPS_DIR)/src/onnxruntime
+else
+    CGO_CPPFLAGS :=
+    CGO_LDFLAGS :=
+    LIBRARY_PATH :=
+endif
+
 #--------------------------------------
 
 ifeq ($(call bool,$(BUILD_STATIC)),1)
@@ -248,7 +278,7 @@ endif
 #--------------------------------------
 
 .PHONY: build
-build: build-info crowdsec cscli plugins  ## Build crowdsec, cscli and plugins
+build: build-info download-deps crowdsec cscli plugins  ## Build crowdsec, cscli and plugins
 
 .PHONY: build-info
 build-info:  ## Print build information
@@ -276,6 +306,29 @@ endif
 .PHONY: all
 all: clean test build  ## Clean, test and build (requires localstack)
 
+
+.PHONY: download-deps
+download-deps: $(DEPS_FILES)
+
+$(DEPS_DIR)/lib/libtokenizers.a:
+	curl --fail -L --output $@ --create-dirs \
+		https://github.com/crowdsecurity/packaging-onnx/releases/download/test/libtokenizers.a \
+
+$(DEPS_DIR)/lib/libonnxruntime.a:
+	curl --fail -L --output $@ --create-dirs \
+		https://github.com/crowdsecurity/packaging-onnx/releases/download/test/libonnxruntime.a \
+
+$(DEPS_DIR)/src/onnxruntime:
+	git clone --depth 1 https://github.com/microsoft/onnxruntime $(DEPS_DIR)/src/onnxruntime -b v1.19.2
+
+# Full list of flags that are passed down to the sub-makefiles in cmd/
+
+MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)" CGO_CPPFLAGS="$(CGO_CPPFLAGS)" LIBRARY_PATH="$(LIBRARY_PATH)"
+
+.PHONY: clean-deps
+clean-deps:
+	@$(RM) -r $(DEPS_DIR)
+
 .PHONY: plugins
 plugins:  ## Build notification plugins
 	@$(foreach plugin,$(PLUGINS), \
@@ -301,7 +354,7 @@ clean-rpm:
 	@$(RM) -r rpm/SRPMS
 
 .PHONY: clean
-clean: clean-debian clean-rpm bats-clean  ## Remove build artifacts
+clean: clean-debian clean-rpm clean-deps bats-clean  ## Remove build artifacts
 	@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
 	@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
 	@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)

build/docker/Dockerfile.debian

@@ -1,6 +1,23 @@
+FROM rust:1.70.0-bullseye AS rust_build
+
+WORKDIR /
+
+RUN apt-get update && \
+    apt-get install -y -q \
+    build-essential \
+    curl \
+    git \
+    make
+
+RUN git clone https://github.com/daulet/tokenizers.git /tokenizer && \
+    cd /tokenizer && \
+    cargo build --release && \
+    cp target/release/libtokenizers.a /tokenizer/libtokenizers.a
+
 FROM docker.io/golang:1.25-bookworm AS build
 
 ARG BUILD_VERSION
+ARG ONNXRUNTIME_VERSION=1.18.1
 
 WORKDIR /go/src/crowdsec
 
@@ -24,7 +41,36 @@ RUN apt-get update && \
 
 COPY . .
 
-RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
+COPY --from=rust_build /tokenizer/libtokenizers.a /usr/local/lib/
+
+# INSTALL ONNXRUNTIME
+RUN cd /tmp && \
+    wget -O onnxruntime.tgz https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-aarch64-${ONNXRUNTIME_VERSION}.tgz && \
+    tar -C /tmp -xvf onnxruntime.tgz && \
+    mv onnxruntime-linux-aarch64-${ONNXRUNTIME_VERSION} onnxruntime && \
+    rm -rf onnxruntime.tgz && \
+    cp -R onnxruntime/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION} /usr/local/lib && \
+    cp onnxruntime/include/*.h /usr/local/include && \
+    rm -rf onnxruntime
+
+RUN ln -s /usr/local/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION} /usr/local/lib/libonnxruntime.so
+
+RUN ls -la /usr/local/include
+RUN ls -la /usr/local/lib
+
+RUN ldconfig
+
+# Test if linking works with a simple program
+RUN echo "#include <onnxruntime_c_api.h>" > test.c && \
+    echo "int main() { return 0; }" >> test.c && \
+    gcc test.c -L/usr/local/lib -lonnxruntime -o test_executable && ./test_executable
+
+RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=0 \
+    CGO_CFLAGS="-D_LARGEFILE64_SOURCE -I/usr/local/include"  \
+    CGO_CPPFLAGS="-I/usr/local/include" \
+    CGO_LDFLAGS="-L/usr/local/lib -lstdc++ -lonnxruntime /usr/local/lib/libtokenizers.a -ldl -lm" \
+    LIBRARY_PATH="/usr/local/lib" \
+    LD_LIBRARY_PATH="/usr/local/lib" && \
     cd crowdsec-v* && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
@@ -42,6 +88,8 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
 
 FROM docker.io/debian:bookworm-slim AS slim
 
+ARG ONNXRUNTIME_VERSION=1.18.1
+
 ENV DEBIAN_FRONTEND=noninteractive
 ENV DEBCONF_NOWARNINGS="yes"
 
@@ -63,6 +111,15 @@ COPY --from=build /go/bin/yq /usr/local/bin/crowdsec /usr/local/bin/cscli /usr/l
 COPY --from=build /etc/crowdsec /staging/etc/crowdsec
 COPY --from=build /go/src/crowdsec/build/docker/docker_start.sh /
 COPY --from=build /go/src/crowdsec/build/docker/config.yaml /staging/etc/crowdsec/config.yaml
+
+# Note Copying this since can't build statically yet
+COPY --from=build /usr/local/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION} /usr/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION}
+COPY --from=build /usr/local/lib/libtokenizers.a /usr/lib/libtokenizers.a
+RUN ln -s /usr/local/lib/libonnxruntime.so.${ONNXRUNTIME_VERSION} /usr/lib/libonnxruntime.so
+COPY --from=build /usr/local/lib/libre2.* /usr/lib/
+
+RUN ls -la /usr/lib
+
 RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml && \
     yq eval -i ".plugin_config.group = \"nogroup\"" /staging/etc/crowdsec/config.yaml
 

go.mod

@@ -28,8 +28,10 @@ require (
 	github.com/corazawaf/libinjection-go v0.2.2
 	github.com/crowdsecurity/dlog v0.0.2
 	github.com/crowdsecurity/go-cs-lib v0.0.24
+	github.com/crowdsecurity/go-onnxruntime v0.0.0-20240801073851-3fd7de0127b4
 	github.com/crowdsecurity/grokky v0.2.2
 	github.com/crowdsecurity/machineid v1.0.3
+	github.com/daulet/tokenizers v0.9.0
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/expr-lang/expr v1.17.7
 	github.com/fatih/color v1.18.0

go.sum

@@ -135,12 +135,16 @@ github.com/crowdsecurity/dlog v0.0.2 h1:nj/7jLKO0o8tYn79O+g51ASeGLr5oOVahSoJ6Umq
 github.com/crowdsecurity/dlog v0.0.2/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
 github.com/crowdsecurity/go-cs-lib v0.0.24 h1:ZIYXHRHCFyByZmMg7S4XE8c/ZMtsTCPVUJbnDcxpTtk=
 github.com/crowdsecurity/go-cs-lib v0.0.24/go.mod h1:X0GMJY2CxdA1S09SpuqIKaWQsvRGxXmecUp9cP599dE=
+github.com/crowdsecurity/go-onnxruntime v0.0.0-20240801073851-3fd7de0127b4 h1:CwzISIxoKp0dJLrJJIlhvQPuzirpS9QH07guxK5LIeg=
+github.com/crowdsecurity/go-onnxruntime v0.0.0-20240801073851-3fd7de0127b4/go.mod h1:YfyL16lx2wA8Z6t/TG1x1/FBngOIpuCuo7nM/FSuP54=
 github.com/crowdsecurity/grokky v0.2.2 h1:yALsI9zqpDArYzmSSxfBq2dhYuGUTKMJq8KOEIAsuo4=
 github.com/crowdsecurity/grokky v0.2.2/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.3 h1:mgd//PhMJqyA1EdRTgwRvafwbzNjoktdJyEgZGVCD2Q=
 github.com/crowdsecurity/machineid v1.0.3/go.mod h1:XWUSlnS0R0+u/JK5ulidwlbceNT3ZOCKteoVQEn6Luo=
 github.com/crowdsecurity/time v0.13.0-crowdsec.20250912 h1:O+lHeYhtRPubKvqDxhuZSjxefd8RbV1Ik5J7hDthoIA=
 github.com/crowdsecurity/time v0.13.0-crowdsec.20250912/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
+github.com/daulet/tokenizers v0.9.0 h1:PSjFUGeuhqb3C0GKP9hdvtHvJ6L1AZceV+0nYGACtCk=
+github.com/daulet/tokenizers v0.9.0/go.mod h1:tGnMdZthXdcWY6DGD07IygpwJqiPvG85FQUnhs/wSCs=
 github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

pkg/cwversion/component/component.go

@@ -19,10 +19,11 @@ var Built = map[string]bool{
 	"datasource_loki":         false,
 	"datasource_s3":           false,
 	"datasource_syslog":       false,
-	"datasource_wineventlog":  false,
 	"datasource_victorialogs": false,
+	"datasource_wineventlog":  false,
 	"datasource_http":         false,
 	"cscli_setup":             false,
+	"mlsupport":               false,
 	"db_mysql":                false,
 	"db_postgres":             false,
 	"db_sqlite":               false,

pkg/exprhelpers/anomalydetection.go

@@ -0,0 +1,57 @@
+//go:build !no_mlsupport
+
+package exprhelpers
+
+import (
+	"errors"
+	"fmt"
+	"log"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion/component"
+	"github.com/crowdsecurity/crowdsec/pkg/ml"
+)
+
+var robertaInferencePipeline *ml.RobertaClassificationInferencePipeline
+
+//nolint:gochecknoinits
+func init() {
+	component.Register("mlsupport")
+}
+
+func InitRobertaInferencePipeline(modelBundlePath string) error {
+	var err error
+
+	fmt.Println("Initializing Roberta Inference Pipeline")
+
+	robertaInferencePipeline, err = ml.NewRobertaInferencePipeline(modelBundlePath)
+	if err != nil {
+		return err
+	}
+	if robertaInferencePipeline == nil {
+		fmt.Println("Failed to initialize Roberta Inference Pipeline")
+	}
+
+	return nil
+}
+
+func IsAnomalous(params ...any) (any, error) {
+	verb, ok1 := params[0].(string)
+	httpPath, ok2 := params[1].(string)
+
+	if !ok1 || !ok2 {
+		return nil, errors.New("parameters must be strings")
+	}
+
+	text := verb + " " + httpPath
+	log.Println("Verb : ", verb)
+	log.Println("HTTP Path : ", httpPath)
+	log.Println("Text to analyze for Anomaly: ", text)
+
+	if robertaInferencePipeline == nil {
+		return nil, errors.New("Roberta Inference Pipeline not properly initialized")
+	}
+
+	result, err := robertaInferencePipeline.PredictLabel(text)
+	boolean_label := result == 1
+	return boolean_label, err
+}

pkg/exprhelpers/anomalydetection_stub.go

@@ -0,0 +1,29 @@
+//go:build no_mlsupport
+
+package exprhelpers
+
+import (
+	"errors"
+	"fmt"
+)
+
+var robertaInferencePipeline *RobertaInferencePipelineStub
+
+type RobertaInferencePipelineStub struct{}
+
+func InitRobertaInferencePipeline(modelBundlePath string) error {
+	fmt.Println("Stub: InitRobertaInferencePipeline called with no ML support")
+	return nil
+}
+
+func IsAnomalous(params ...any) (any, error) {
+	_, ok1 := params[0].(string)
+	_, ok2 := params[1].(string)
+
+	if !ok1 || !ok2 {
+		return nil, errors.New("parameters must be strings")
+	}
+	fmt.Println("IsAnomalous: InitRobertaInferencePipeline called with no ML support")
+
+	return false, nil
+}