Merge branch 'main' into cherrypy-instrumentation

.flake8

@@ -16,6 +16,7 @@ exclude =
   target
   __pycache__
   exporter/opentelemetry-exporter-jaeger/src/opentelemetry/exporter/jaeger/gen/
+  exporter/opentelemetry-exporter-prometheus-remote-write/src/opentelemetry/exporter/prometheus_remote_write/gen/
   exporter/opentelemetry-exporter-jaeger/build/*
   docs/examples/opentelemetry-example-app/src/opentelemetry_example_app/grpc/gen/
   docs/examples/opentelemetry-example-app/build/*

.github/workflows/test.yml

@@ -6,7 +6,7 @@ on:
     - 'release/*'
   pull_request:
 env:
-  CORE_REPO_SHA: 321f90f96b80dc8118d58cc639e26a950418d138
+  CORE_REPO_SHA: a97b3a70e852f77c6a25c69192fc506c127cdaaa
 
 jobs:
   build:
@@ -16,13 +16,14 @@ jobs:
       py38: 3.8
       py39: 3.9
       py310: "3.10"
+      py311: "3.11"
       pypy3: "pypy3.7"
       RUN_MATRIX_COMBINATION: ${{ matrix.python-version }}-${{ matrix.package }}-${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false  # ensures the entire test matrix is run, even if one permutation fails
       matrix:
-        python-version: [ py37, py38, py39, py310, pypy3 ]
+        python-version: [ py37, py38, py39, py310, py311, pypy3 ]
         package: ["instrumentation", "distro", "exporter", "sdkextension", "propagator"]
         os: [ ubuntu-20.04 ]
     steps:

RELEASING.md

@@ -0,0 +1,102 @@
+# Releasing OpenTelemetry Packages (for maintainers only)
+This document explains how to publish all OT modules at version x.y.z. Ensure that you’re following semver when choosing a version number.
+
+Release Process:
+* [Checkout a clean repo](#checkout-a-clean-repo)
+* [Update versions](#update-versions)
+* [Create a new branch](#create-a-new-branch)
+* [Open a Pull Request](#open-a-pull-request)
+* [Create a Release](#Create-a-Release)
+* [Move stable tag](#Move-stable-tag)
+* [Update main](#Update-main)
+* [Check PyPI](#Check-PyPI)
+* [Troubleshooting](#troubleshooting)
+
+## Checkout a clean repo
+To avoid pushing untracked changes, check out the repo in a new dir
+
+## Update versions
+The update of the version information relies on the information in eachdist.ini to identify which packages are stable, prerelease or
+experimental. Update the desired version there to begin the release process.
+
+## Create a new branch
+The following script does the following:
+- update main locally
+- creates a new release branch `release/<version>`
+- updates version and changelog files
+- commits the change
+
+*NOTE: This script was run by a GitHub Action but required the Action bot to be excluded from the CLA check, which it currently is not.*
+
+```bash
+./scripts/prepare_release.sh
+```
+
+## Open a Pull Request
+
+The PR should be opened from the `release/<version>` branch created as part of running `prepare_release.sh` in the steps above.
+
+## Create a Release
+
+- Create the GH release from the main branch, using a new tag for this micro version, e.g. `v0.7.0`
+- Copy the changelogs from all packages that changed into the release notes (and reformat to remove hard line wraps)
+
+
+## Check PyPI
+
+This should be handled automatically on release by the [publish action](https://github.com/open-telemetry/opentelemetry-python/blob/main/.github/workflows/publish.yml).
+
+- Check the [action logs](https://github.com/open-telemetry/opentelemetry-python/actions?query=workflow%3APublish) to make sure packages have been uploaded to PyPI
+- Check the release history (e.g. https://pypi.org/project/opentelemetry-api/#history) on PyPI
+
+If for some reason the action failed, see [Publish failed](#publish-failed) below
+
+## Move stable tag
+
+This will ensure the docs are pointing at the stable release.
+
+```bash
+git tag -d stable
+git tag stable
+git push --delete origin tagname
+git push origin stable
+```
+
+To validate this worked, ensure the stable build has run successfully: https://readthedocs.org/projects/opentelemetry-python/builds/. If the build has not run automatically, it can be manually trigger via the readthedocs interface.
+
+## Update main
+
+Ensure the version and changelog updates have been applied to main. Update the versions in eachdist.ini once again this time to include the `.dev0` tag and
+run eachdist once again:
+```bash
+./scripts/eachdist.py update_versions --versions stable,prerelease
+```
+
+If the diff includes significant changes, create a pull request to commit the changes and once the changes are merged, click the "Run workflow" button for the Update [OpenTelemetry Website Docs](https://github.com/open-telemetry/opentelemetry-python/actions/workflows/docs-update.yml) GitHub Action.
+
+## Hotfix procedure
+
+A `hotfix` is defined as a small change developed to correct a bug that should be released as quickly as possible. Due to the nature of hotfixes, they usually will only affect one or a few packages. Therefore, it usually is not necessary to go through the entire release process outlined above for hotfixes. Follow the below steps how to release a hotfix:
+
+1. Identify the packages that are affected by the bug. Make the changes to those packages, merging to `main`, as quickly as possible.
+2. On your local machine, remove the `dev0` tags from the version number and increment the patch version number.
+3. On your local machine, update `CHANGELOG.md` with the date of the hotfix change.
+4. With administrator privileges for PyPi, manually publish the affected packages.
+    a. Install [twine](https://pypi.org/project/twine/)
+    b. Navigate to where the `setup.py` file exists for the package you want to publish.
+    c. Run `python setup.py sdist bdist_wheel`. You may have to install [wheel](https://pypi.org/project/wheel/) as well.
+    d. Validate your built distributions by running `twine check dist/*`.
+    e. Upload distributions to PyPi by running `twine upload dist/*`.
+5. Note that since hotfixes are manually published, the build scripts for publish after creating a release are not run.
+
+## Troubleshooting
+
+### Publish failed
+
+If for some reason the action failed, do it manually:
+
+- Switch to the release branch (important so we don't publish packages with "dev" versions)
+- Build distributions with `./scripts/build.sh`
+- Delete distributions we don't want to push (e.g. `testutil`)
+- Push to PyPI as `twine upload --skip-existing --verbose dist/*`
+- Double check PyPI!

_template/pyproject.toml

@@ -26,6 +26,7 @@ classifiers = [
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
 ]
 dependencies = [
   "opentelemetry-api ~= 1.12",

_template/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.34b0"
+__version__ = "0.36b0.dev"

eachdist.ini

@@ -16,7 +16,7 @@ sortfirst=
     ext/*
 
 [stable]
-version=1.13.0
+version=1.15.0.dev
 
 packages=
     opentelemetry-sdk
@@ -34,7 +34,7 @@ packages=
     opentelemetry-api
 
 [prerelease]
-version=0.34b0
+version=0.36b0.dev
 
 packages=
     all

exporter/opentelemetry-exporter-prometheus-remote-write/README.rst

@@ -0,0 +1,29 @@
+OpenTelemetry Prometheus Remote Write Exporter
+==============================================
+
+|pypi|
+
+.. |pypi| image:: https://badge.fury.io/py/opentelemetry-exporter-prometheus-remote-write.svg
+   :target: https://pypi.org/project/opentelemetry-exporter-prometheus-remote-write/
+
+This package contains an exporter to send metrics from the OpenTelemetry Python SDK directly to a Prometheus Remote Write integrated backend
+(such as Cortex or Thanos) without having to run an instance of the Prometheus server.
+
+
+Installation
+------------
+
+::
+
+    pip install opentelemetry-exporter-prometheus-remote-write
+
+
+.. _OpenTelemetry: https://github.com/open-telemetry/opentelemetry-python/
+.. _Prometheus Remote Write integrated backend: https://prometheus.io/docs/operating/integrations/
+
+
+References
+----------
+
+* `OpenTelemetry Project <https://opentelemetry.io/>`_
+* `Prometheus Remote Write Integration <https://prometheus.io/docs/operating/integrations/>`_

exporter/opentelemetry-exporter-prometheus-remote-write/example/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.8
+
+RUN apt-get update -y && apt-get install libsnappy-dev -y
+
+WORKDIR /code
+COPY . .
+
+RUN pip install -e .
+RUN pip install -r ./examples/requirements.txt
+
+CMD ["python", "./examples/sampleapp.py"]

exporter/opentelemetry-exporter-prometheus-remote-write/example/README.md

@@ -0,0 +1,42 @@
+# Prometheus Remote Write Exporter Example
+This example uses [Docker Compose](https://docs.docker.com/compose/) to set up:
+
+1. A Python program that creates 5 instruments with 5 unique
+aggregators and a randomized load generator
+2. An instance of [Cortex](https://cortexmetrics.io/) to receive the metrics
+data
+3. An instance of [Grafana](https://grafana.com/) to visualizse the exported
+data
+
+## Requirements
+* Have Docker Compose [installed](https://docs.docker.com/compose/install/)
+
+*Users do not need to install Python as the app will be run in the Docker Container*
+
+## Instructions
+1. Run `docker-compose up -d` in the the `examples/` directory
+
+The `-d` flag causes all services to run in detached mode and frees up your
+terminal session. This also causes no logs to show up. Users can attach themselves to the service's logs manually using `docker logs ${CONTAINER_ID} --follow`
+
+2. Log into the Grafana instance at [http://localhost:3000](http://localhost:3000)
+   * login credentials are `username: admin` and `password: admin`
+   * There may be an additional screen on setting a new password. This can be skipped and is optional
+
+3. Navigate to the `Data Sources` page
+   * Look for a gear icon on the left sidebar and select `Data Sources`
+
+4. Add a new Prometheus Data Source
+   * Use `http://cortex:9009/api/prom` as the URL
+   * (OPTIONAl) set the scrape interval to `2s` to make updates appear quickly
+   * click `Save & Test`
+
+5. Go to `Metrics Explore` to query metrics
+   * Look for a compass icon on the left sidebar
+   * click `Metrics` for a dropdown list of all the available metrics
+   * (OPTIONAL) Adjust time range by clicking the `Last 6 hours` button on the upper right side of the graph
+   * (OPTIONAL) Set up auto-refresh by selecting an option under the dropdown next to the refresh button on the upper right side of the graph
+   * Click the refresh button and data should show up on the graph
+
+6. Shutdown the services when finished
+   * Run `docker-compose down` in the examples directory

exporter/opentelemetry-exporter-prometheus-remote-write/example/cortex-config.yml

@@ -0,0 +1,101 @@
+# This Cortex Config is copied from the Cortex Project documentation
+# Source: https://github.com/cortexproject/cortex/blob/master/docs/configuration/single-process-config.yaml
+
+# Configuration for running Cortex in single-process mode.
+# This configuration should not be used in production.
+# It is only for getting started and development.
+
+# Disable the requirement that every request to Cortex has a
+# X-Scope-OrgID header. `fake` will be substituted in instead.
+# pylint: skip-file
+auth_enabled: false
+
+server:
+  http_listen_port: 9009
+
+  # Configure the server to allow messages up to 100MB.
+  grpc_server_max_recv_msg_size: 104857600
+  grpc_server_max_send_msg_size: 104857600
+  grpc_server_max_concurrent_streams: 1000
+
+distributor:
+  shard_by_all_labels: true
+  pool:
+    health_check_ingesters: true
+
+ingester_client:
+  grpc_client_config:
+    # Configure the client to allow messages up to 100MB.
+    max_recv_msg_size: 104857600
+    max_send_msg_size: 104857600
+    use_gzip_compression: true
+
+ingester:
+  # We want our ingesters to flush chunks at the same time to optimise
+  # deduplication opportunities.
+  spread_flushes: true
+  chunk_age_jitter: 0
+
+  walconfig:
+    wal_enabled: true
+    recover_from_wal: true
+    wal_dir: /tmp/cortex/wal
+
+  lifecycler:
+    # The address to advertise for this ingester.  Will be autodiscovered by
+    # looking up address on eth0 or en0; can be specified if this fails.
+    # address: 127.0.0.1
+
+    # We want to start immediately and flush on shutdown.
+    join_after: 0
+    min_ready_duration: 0s
+    final_sleep: 0s
+    num_tokens: 512
+    tokens_file_path: /tmp/cortex/wal/tokens
+
+    # Use an in memory ring store, so we don't need to launch a Consul.
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+
+# Use local storage - BoltDB for the index, and the filesystem
+# for the chunks.
+schema:
+  configs:
+    - from: 2019-07-29
+      store: boltdb
+      object_store: filesystem
+      schema: v10
+      index:
+        prefix: index_
+        period: 1w
+
+storage:
+  boltdb:
+    directory: /tmp/cortex/index
+
+  filesystem:
+    directory: /tmp/cortex/chunks
+
+  delete_store:
+    store: boltdb
+
+purger:
+  object_store_type: filesystem
+
+frontend_worker:
+  # Configure the frontend worker in the querier to match worker count
+  # to max_concurrent on the queriers.
+  match_max_concurrent: true
+
+# Configure the ruler to scan the /tmp/cortex/rules directory for prometheus
+# rules: https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules
+ruler:
+  enable_api: true
+  enable_sharding: false
+  storage:
+    type: local
+    local:
+      directory: /tmp/cortex/rules
+

exporter/opentelemetry-exporter-prometheus-remote-write/example/docker-compose.yml

@@ -0,0 +1,33 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.8"
+
+services:
+  cortex:
+    image: quay.io/cortexproject/cortex:v1.5.0
+    command:
+      - -config.file=./config/cortex-config.yml
+    volumes:
+      - ./cortex-config.yml:/config/cortex-config.yml:ro
+    ports:
+      - 9009:9009
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - 3000:3000
+  sample_app:
+    build:
+      context: ../
+      dockerfile: ./examples/Dockerfile

exporter/opentelemetry-exporter-prometheus-remote-write/example/requirements.txt

@@ -0,0 +1,7 @@
+psutil
+protobuf>=3.13.0
+requests>=2.25.0
+python-snappy>=0.5.4
+opentelemetry-api
+opentelemetry-sdk
+opentelemetry-proto