Skip to content

Commit

Permalink
Manual instrumentation support - Phase 1 (#523)
Browse files Browse the repository at this point in the history
* Initial instrumentation of otel sdk trace functions

* Add offsets for otel go sdk

* Pass the user defined span name through eBPF

* Initial attribute parsing

* Fix struct for attribute value

* Try more efficient attribute encoding, currently only working for numeric values, missing Go decoding

* Initial draft

* Instrument Otel API functions to integrate manual spans with automatic ones

* revert changes to verifier log collection settings

* Add tests for otel API instrumentation

* update changelog and lint

* Check kernel version

* Change format of attributes in eBPF to make the verification easier

* Small fix

* Adding printk if attribute is too long for buffer

* Update internal/include/otel_types.h

Co-authored-by: Mike Goldsmith <goldsmith.mike@gmail.com>

* Code review 1

* Apply suggestions from code review

Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com>

* rename WithOtelApi to globalImpl

* Rename probe folder

* Inject attribute types consts from Go

* add cli flag for otel-global to record telemetry from the OpenTelemetry default global

* debug tests

* fix debug

* modify set_attr_value

* print verifier log

* larger verifier log

* ...

* simplify eBPF code

* Clean-up and updating changelog

* run make precommit

* Fix doc

* Apply suggestions from code review

---------

Co-authored-by: Mike Goldsmith <goldsmith.mike@gmail.com>
Co-authored-by: Tyler Yahn <MrAlias@users.noreply.github.com>
  • Loading branch information
3 people authored Dec 7, 2023
1 parent 2d37cd2 commit dc3681a
Show file tree
Hide file tree
Showing 20 changed files with 1,337 additions and 10 deletions.
9 changes: 9 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,15 @@ updates:
schedule:
interval: weekly
day: sunday
- package-ecosystem: docker
directory: /internal/test/e2e/otelglobal
labels:
- dependencies
- docker
- Skip Changelog
schedule:
interval: weekly
day: sunday
- package-ecosystem: gomod
directory: /
labels:
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/e2e/k8s/sample-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ spec:
- name: auto-instrumentation
image: otel-go-instrumentation
imagePullPolicy: IfNotPresent
command: ["/otel-go-instrumentation", "-global-impl"]
env:
- name: OTEL_GO_AUTO_TARGET_EXE
value: /sample-app/main
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/kind.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
strategy:
matrix:
k8s-version: ["v1.26.0"]
library: ["nethttp", "nethttp_custom", "gin", "databasesql", "grpc"]
library: ["nethttp", "nethttp_custom", "gin", "databasesql", "grpc", "otelglobal"]
runs-on: ubuntu-latest
steps:
- name: Checkout Repo
Expand Down Expand Up @@ -68,7 +68,7 @@ jobs:
kubectl -n default create -f .github/workflows/e2e/k8s/sample-job.yml
- name: check job status
run: |
kubectl wait --for=condition=Complete --timeout=60s job/sample-job
kubectl wait --for=condition=Complete --timeout=60s job/sample-job || kubectl logs -l app=sample -c auto-instrumentation
- name: copy telemetry trace output
run: |
kubectl cp -c filecp default/test-opentelemetry-collector-0:tmp/trace.json ./internal/test/e2e/${{ matrix.library }}/traces-orig.json
Expand Down
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ OpenTelemetry Go Automatic Instrumentation adheres to [Semantic Versioning](http

### Added

- The CLI flag `global-impl` is added.
This flag, when used, enables the instrumentation of the OpenTelemetry default global implementation (https://pkg.go.dev/go.opentelemetry.io/otel).
This means that all trace telemetry from this implementation that would normally be dropped will instead be recorded with the auto-instrumentation pipeline. ([#523]https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/523)
- Add `WithResourceAttributes` `InstrumentationOption` to configure `Instrumentation` to add additional resource attributes. ([#522](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/522))

### Changed
Expand Down
5 changes: 3 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -142,12 +142,13 @@ license-header-check:
exit 1; \
fi

.PHONY: fixture-nethttp fixture-gin fixture-databasesql fixture-nethttp-custom
.PHONY: fixture-nethttp fixture-gin fixture-databasesql fixture-nethttp-custom fixture-otelglobal
fixture-nethttp-custom: fixtures/nethttp_custom
fixture-nethttp: fixtures/nethttp
fixture-gin: fixtures/gin
fixture-databasesql: fixtures/databasesql
fixture-grpc: fixtures/grpc
fixture-otelglobal: fixtures/otelglobal
fixtures/%: LIBRARY=$*
fixtures/%:
$(MAKE) docker-build
Expand All @@ -161,7 +162,7 @@ fixtures/%:
helm install test -f .github/workflows/e2e/k8s/collector-helm-values.yml opentelemetry-helm-charts/charts/opentelemetry-collector
kubectl wait --for=condition=Ready --timeout=60s pod/test-opentelemetry-collector-0
kubectl -n default create -f .github/workflows/e2e/k8s/sample-job.yml
kubectl wait --for=condition=Complete --timeout=60s job/sample-job
kubectl wait --for=condition=Complete --timeout=60s job/sample-job || kubectl logs -l app=sample -c auto-instrumentation
kubectl cp -c filecp default/test-opentelemetry-collector-0:tmp/trace.json ./internal/test/e2e/$(LIBRARY)/traces-orig.json
rm -f ./internal/test/e2e/$(LIBRARY)/traces.json
bats ./internal/test/e2e/$(LIBRARY)/verify.bats
Expand Down
13 changes: 11 additions & 2 deletions cli/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@ func newLogger() logr.Logger {
}

func main() {
var globalImpl bool

flag.BoolVar(&globalImpl, "global-impl", false, "Record telemetry from the OpenTelemetry default global implementation")
flag.Usage = usage
flag.Parse()

Expand All @@ -88,8 +91,14 @@ func main() {
}
}()

logger.Info("building OpenTelemetry Go instrumentation ...")
inst, err := auto.NewInstrumentation(ctx, auto.WithEnv())
logger.Info("building OpenTelemetry Go instrumentation ...", "globalImpl", globalImpl)

instOptions := []auto.InstrumentationOption{auto.WithEnv()}
if globalImpl {
instOptions = append(instOptions, auto.WithGlobal())
}

inst, err := auto.NewInstrumentation(ctx, instOptions...)
if err != nil {
logger.Error(err, "failed to create instrumentation")
return
Expand Down
5 changes: 4 additions & 1 deletion examples/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,7 @@ require (
go.uber.org/zap v1.26.0
)

require go.uber.org/multierr v1.10.0 // indirect
require (
github.com/stretchr/testify v1.8.4 // indirect
go.uber.org/multierr v1.10.0 // indirect
)
3 changes: 2 additions & 1 deletion examples/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
Expand Down
27 changes: 26 additions & 1 deletion instrumentation.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ func NewInstrumentation(ctx context.Context, opts ...InstrumentationOption) (*In
return nil, err
}

mngr, err := instrumentation.NewManager(logger, ctrl)
mngr, err := instrumentation.NewManager(logger, ctrl, c.globalImpl)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -170,6 +170,7 @@ type instConfig struct {
target process.TargetArgs
serviceName string
additionalResAttrs []attribute.KeyValue
globalImpl bool
}

func newInstConfig(ctx context.Context, opts []InstrumentationOption) (instConfig, error) {
Expand Down Expand Up @@ -417,6 +418,30 @@ func WithSampler(sampler trace.Sampler) InstrumentationOption {
})
}

// WithGlobal returns an [InstrumentationOption] that will configure an
// [Instrumentation] to record telemetry from the [OpenTelemetry default global
// implementation]. By default, the OpenTelemetry global implementation is a
// no-op implementation of the OpenTelemetry API. However, by using this
// option, all telemetry that would have been dropped by the global
// implementation will be recorded using telemetry pipelines from the
// configured [Instrumentation].
//
// If the target process overrides the default global implementation (e.g.
// [otel.SetTracerProvider]), the telemetry from that process will go to the
// set implementation. It will not be recorded using the telemetry pipelines
// from the configured [Instrumentation] even if this option is used.
//
// The OpenTelemetry default global implementation is left unchanged (i.e. it
// remains a no-op implementation) if this options is not used.
//
// [OpenTelemetry default global implementation]: https://pkg.go.dev/go.opentelemetry.io/otel
func WithGlobal() InstrumentationOption {
return fnOpt(func(_ context.Context, c instConfig) (instConfig, error) {
c.globalImpl = true
return c, nil
})
}

// WithResourceAttributes returns an [InstrumentationOption] that will configure
// an [Instrumentation] to add the provided attributes to the OpenTelemetry resource.
func WithResourceAttributes(attrs ...attribute.KeyValue) InstrumentationOption {
Expand Down
143 changes: 143 additions & 0 deletions internal/include/otel_types.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef _OTEL_TYPES_H
#define _OTEL_TYPES_H

#include "go_types.h"
#include "common.h"

// Injected in init
volatile const u64 attr_type_invalid;

volatile const u64 attr_type_bool;
volatile const u64 attr_type_int64;
volatile const u64 attr_type_float64;
volatile const u64 attr_type_string;

volatile const u64 attr_type_boolslice;
volatile const u64 attr_type_int64slice;
volatile const u64 attr_type_float64slice;
volatile const u64 attr_type_stringslice;

/* Defintions should mimic structs defined in go.opentelemetry.io/otel/attribute */

typedef struct go_otel_attr_value {
u64 vtype;
u64 numeric;
struct go_string string;
struct go_iface slice;
} go_otel_attr_value_t;

typedef struct go_otel_key_value {
struct go_string key;
go_otel_attr_value_t value;
} go_otel_key_value_t;

#define OTEL_ATTRIBUTE_KEY_MAX_LEN (32)
#define OTEL_ATTRIBUTE_VALUE_MAX_LEN (128)
#define OTEL_ATTRUBUTE_MAX_COUNT (16)

typedef struct otel_attirbute {
u16 val_length;
u8 vtype;
u8 reserved;
char key[OTEL_ATTRIBUTE_KEY_MAX_LEN];
char value[OTEL_ATTRIBUTE_VALUE_MAX_LEN];
} otel_attirbute_t;

typedef struct otel_attributes {
otel_attirbute_t attrs[OTEL_ATTRUBUTE_MAX_COUNT];
u8 valid_attrs;
}__attribute__((packed)) otel_attributes_t;

static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_value_t *go_attr_value)
{
u64 vtype = go_attr_value->vtype;

if (vtype == attr_type_invalid) {
bpf_printk("Invalid attribute value type\n");
return false;
}

// Constant size values
if (vtype == attr_type_bool ||
vtype == attr_type_int64 ||
vtype == attr_type_float64) {
bpf_probe_read(attr->value, sizeof(s64), &go_attr_value->numeric);
return true;
}

// String values
if (vtype == attr_type_string) {
if (go_attr_value->string.len >= OTEL_ATTRIBUTE_VALUE_MAX_LEN) {
bpf_printk("Aattribute string value is too long\n");
return false;
}
return get_go_string_from_user_ptr(&go_attr_value->string, attr->value, OTEL_ATTRIBUTE_VALUE_MAX_LEN);
}

// TODO (#525): handle slices
return false;
}

static __always_inline void convert_go_otel_attributes(void *attrs_buf, s64 slice_len, otel_attributes_t *enc_attrs)
{
if (attrs_buf == NULL || enc_attrs == NULL){
return;
}

if (slice_len < 1) {
return;
}

s64 num_attrs = slice_len < OTEL_ATTRUBUTE_MAX_COUNT ? slice_len : OTEL_ATTRUBUTE_MAX_COUNT;
go_otel_key_value_t *go_attr = (go_otel_key_value_t*)attrs_buf;
go_otel_attr_value_t go_attr_value = {0};
struct go_string go_str = {0};
u8 valid_attrs = 0;

for (u32 go_attr_index = 0; go_attr_index < num_attrs; go_attr_index++) {
__builtin_memset(&go_attr_value, 0, sizeof(go_otel_attr_value_t));
// Read the value struct
bpf_probe_read(&go_attr_value, sizeof(go_otel_attr_value_t), &go_attr[go_attr_index].value);

if (go_attr_value.vtype == attr_type_invalid) {
continue;
}

// Read the key string
bpf_probe_read(&go_str, sizeof(struct go_string), &go_attr[go_attr_index].key);
if (go_str.len >= OTEL_ATTRIBUTE_KEY_MAX_LEN) {
// key string is too large
bpf_printk("Attribute key string is too long\n");
continue;
}

if (!get_go_string_from_user_ptr(&go_str, enc_attrs->attrs[valid_attrs].key, OTEL_ATTRIBUTE_KEY_MAX_LEN)) {
continue;
}

if (!set_attr_value(&enc_attrs->attrs[valid_attrs], &go_attr_value)) {
continue;
}

enc_attrs->attrs[valid_attrs].vtype = go_attr_value.vtype;
valid_attrs++;
}

enc_attrs->valid_attrs = valid_attrs;
}

#endif
Loading

0 comments on commit dc3681a

Please sign in to comment.