diff --git a/Cargo.lock b/Cargo.lock index 8e11726b8577a..76649b716c9bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8028,7 +8028,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.11.0", + "itertools 0.10.5", "log", "multimap", "once_cell", @@ -8074,7 +8074,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.10.5", "proc-macro2 1.0.95", "quote 1.0.40", "syn 2.0.104", @@ -12370,7 +12370,7 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" version = "0.25.0" -source = "git+https://github.com/vectordotdev/vrl.git?branch=main#23c44aeeddaffc0226b164ac951bd075d2972361" +source = "git+https://github.com/vectordotdev/vrl.git?branch=main#834e80187f508a48b06b4ea7cf0acf7dae81ebc9" dependencies = [ "aes", "aes-siv", diff --git a/website/cue/reference/remap/functions/decode_lz4.cue b/website/cue/reference/remap/functions/decode_lz4.cue index 9f9487444e1ae..25baf7d14b600 100644 --- a/website/cue/reference/remap/functions/decode_lz4.cue +++ b/website/cue/reference/remap/functions/decode_lz4.cue @@ -3,7 +3,9 @@ package metadata remap: functions: decode_lz4: { category: "Codec" description: """ - Decodes the `value` (an lz4 string) into its original string. + Decodes the `value` (an lz4 string) into its original string. `buf_size` is the size of the buffer to decode into, this must be equal to or larger than the uncompressed size. + If `prepended_size` is set to `true`, it expects the original uncompressed size to be prepended to the compressed data. + `prepended_size` is useful for some implementations of lz4 that require the original size to be known before decoding. """ arguments: [ @@ -13,17 +15,42 @@ remap: functions: decode_lz4: { required: true type: ["string"] }, + { + name: "buf_size" + description: "The size of the buffer to decode into, this must be equal to or larger than the uncompressed size." + required: false + default: 1024 * 1024 // 1 MiB + type: ["integer"] + }, + { + name: "prepended_size" + description: "Some implementations of lz4 require the original uncompressed size to be prepended to the compressed data." + required: false + default: false + type: ["boolean"] + }, ] internal_failure_reasons: [ - "`value` unable to decode value with lz4 decoder.", + "`value` unable to decode value with lz4 frame decoder.", + "`value` unable to decode value with lz4 block decoder.", + "`value` unable to decode because the output is too large for the buffer.", + "`value` unable to decode because the prepended size is not a valid integer.", ] return: types: ["string"] examples: [ { - title: "Decode Lz4 data" + title: "Decode Lz4 data with prepended size." source: #""" encoded_text = decode_base64!("LAAAAPAdVGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIDEzIGxhenkgZG9ncy4=") + decode_lz4!(encoded_text, use_prepended_size: true) + """# + return: "The quick brown fox jumps over 13 lazy dogs." + }, + { + title: "Decode Lz4 data without prepended size." + source: #""" + encoded_text = decode_base64!("8B1UaGUgcXVpY2sgYnJvd24gZm94IGp1bXBzIG92ZXIgMTMgbGF6eSBkb2dzLg==") decode_lz4!(encoded_text) """# return: "The quick brown fox jumps over 13 lazy dogs." diff --git a/website/cue/reference/remap/functions/encode_lz4.cue b/website/cue/reference/remap/functions/encode_lz4.cue index a8761c9598666..66ebe7e45bcd9 100644 --- a/website/cue/reference/remap/functions/encode_lz4.cue +++ b/website/cue/reference/remap/functions/encode_lz4.cue @@ -3,7 +3,9 @@ package metadata remap: functions: encode_lz4: { category: "Codec" description: """ - Encodes the `value` to [Lz4](\(urls.lz4)). + Encodes the `value` to [Lz4](\(urls.lz4)). This function compresses the input string into an lz4 block. + If `prepend_size` is set to `true`, it prepends the original uncompressed size to the compressed data. + This is useful for some implementations of lz4 that require the original size to be known before decoding. """ arguments: [ @@ -13,6 +15,13 @@ remap: functions: encode_lz4: { required: true type: ["string"] }, + { + name: "prepend_size" + description: "Whether to prepend the original size to the compressed data." + required: false + default: false + type: ["boolean"] + }, ] internal_failure_reasons: [] return: types: ["string"]