diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f0f3e4ca..e9564608 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,59 +7,67 @@ jobs: strategy: fail-fast: false matrix: - cassandra-version: [3.11, 4.0] + cassandra-version: [3.11, 4.0, 4.1] include: - cassandra-version: 3.11 run311tests: true run40tests: false + run41tests: false - cassandra-version: 4.0 run311tests: false run40tests: true + run41tests: false + - cassandra-version: 4.1 + run311tests: false + run40tests: false + run41tests: true runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 1.8 + java-version: 8 + distribution: 'zulu' - name: Cache Maven packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Setup Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: version: latest - name: Build with Maven and run tests run: | - mvn -B -q install --file pom.xml -Drun311tests=${{ matrix.run311tests }} -Drun40tests=${{ matrix.run40tests }} + mvn -B -q install --file pom.xml -Drun311tests=${{ matrix.run311tests }} -Drun40tests=${{ matrix.run40tests }} -Drun41tests=${{ matrix.run41tests }} build-dse: if: ${{ github.ref == 'refs/heads/master' && github.event_name == 'push'}} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 1.8 + java-version: 8 + distribution: 'zulu' - name: Cache Maven packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Setup Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: version: latest - name: Build with Maven and run tests @@ -96,9 +104,12 @@ jobs: - cassandra-version: '4.0' docker-file: Dockerfile-4_0 build-target: oss40 + - cassandra-version: '4.1' + docker-file: Dockerfile-4_1 + build-target: oss41 steps: - name: Check out source code - uses: actions/checkout@v2 + uses: actions/checkout@v3 # Setup metadata based on the commit/tag that will be used for tagging the image # Only build and publish a commit based tag - name: Setup Docker meta @@ -108,9 +119,9 @@ jobs: images: k8ssandra/cass-management-api tags: type=sha,prefix=${{ matrix.cassandra-version }}- - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub uses: docker/login-action@v1 with: @@ -136,7 +147,7 @@ jobs: jdk-version: [jdk8, jdk11] steps: - name: Check out source code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Maven settings file run: | mkdir -p ~/.m2 @@ -166,7 +177,7 @@ jobs: images: datastax/dse-mgmtapi-6_8 tags: type=sha,prefix=dse68-${{ matrix.jdk-version }}- - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub uses: docker/login-action@v1 with: diff --git a/.github/workflows/docker-release.yaml b/.github/workflows/docker-release.yaml index c1271673..edd256f4 100644 --- a/.github/workflows/docker-release.yaml +++ b/.github/workflows/docker-release.yaml @@ -17,7 +17,7 @@ jobs: latest: true runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Setup Maven Settings File run: | mkdir -p ~/.m2 @@ -39,10 +39,10 @@ jobs: EOF cp ~/.m2/settings.xml settings.xml - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Setup Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: version: latest - name: Login to Docker Hub @@ -104,12 +104,12 @@ jobs: latest: true runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Setup Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: version: latest - name: Login to Docker Hub @@ -148,12 +148,12 @@ jobs: latest: true runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Setup Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 with: version: latest - name: Login to Docker Hub @@ -181,3 +181,47 @@ jobs: --file Dockerfile-4_0 \ --target oss40 \ --platform linux/amd64,linux/arm64 . + + build-oss-4-1x: + strategy: + fail-fast: false + matrix: + cassandra-version: [4.1-beta1] + include: + - cassandra-version: 4.1-beta1 + latest: true + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Setup Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + version: latest + - name: Login to Docker Hub + run: echo "${{ secrets.DOCKER_HUB_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_HUB_USERNAME }}" --password-stdin + - if: ${{ matrix.latest }} + name: Publish ${{ matrix.cassandra-version }} to Registry + run: | + RELEASE_VERSION="${GITHUB_REF##*/}" + docker buildx build --push \ + --build-arg CASSANDRA_VERSION=${{ matrix.cassandra-version }} \ + --tag k8ssandra/cass-management-api:4.1 \ + --tag k8ssandra/cass-management-api:${{ matrix.cassandra-version }} \ + --tag k8ssandra/cass-management-api:${{ matrix.cassandra-version }}-$RELEASE_VERSION \ + --file Dockerfile-4_1 \ + --target oss41 \ + --platform linux/amd64,linux/arm64 . + - if: ${{ !matrix.latest }} + name: Publish ${{ matrix.cassandra-version }} to Registry + run: | + RELEASE_VERSION="${GITHUB_REF##*/}" + docker buildx build --push \ + --build-arg CASSANDRA_VERSION=${{ matrix.cassandra-version }} \ + --tag k8ssandra/cass-management-api:${{ matrix.cassandra-version }} \ + --tag k8ssandra/cass-management-api:${{ matrix.cassandra-version }}-$RELEASE_VERSION \ + --file Dockerfile-4_1 \ + --target oss41 \ + --platform linux/amd64,linux/arm64 . diff --git a/.github/workflows/jar-release.yaml b/.github/workflows/jar-release.yaml index 43684e33..d9765004 100644 --- a/.github/workflows/jar-release.yaml +++ b/.github/workflows/jar-release.yaml @@ -35,12 +35,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Setup Java JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 13 - java-package: jdk + java-version: 8 + distribution: 'zulu' - name: Build with Maven run: | cat < ~/.m2/settings.xml @@ -62,7 +62,7 @@ jobs: mvn -q -ff package -DskipTests -P dse - name: zip-up run: | - zip jars.zip management-api-agent-common/target/datastax-mgmtapi-agent-*.jar management-api-agent-3.x/target/datastax-mgmtapi-agent-*.jar management-api-agent-4.x/target/datastax-mgmtapi-agent-*.jar management-api-agent-dse-6.8/target/datastax-mgmtapi-agent-*.jar management-api-server/target/datastax-mgmtapi-server-*.jar management-api-common/target/datastax-mgmtapi-common-*.jar + zip jars.zip management-api-agent-common/target/datastax-mgmtapi-agent-*.jar management-api-agent-3.x/target/datastax-mgmtapi-agent-*.jar management-api-agent-4.x/target/datastax-mgmtapi-agent-*.jar management-api-agent-4.1.x/target/datastax-mgmtapi-agent-*.jar management-api-agent-dse-6.8/target/datastax-mgmtapi-agent-*.jar management-api-server/target/datastax-mgmtapi-server-*.jar management-api-common/target/datastax-mgmtapi-common-*.jar - name: Retrieve stashed release URL uses: actions/download-artifact@v1 with: diff --git a/.github/workflows/license-check.yaml b/.github/workflows/license-check.yaml index 89f4de33..848b3567 100644 --- a/.github/workflows/license-check.yaml +++ b/.github/workflows/license-check.yaml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Fossa CLI run: | curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/fossa-cli/master/install-latest.sh | bash -s -- -b . diff --git a/.github/workflows/maven-publish.yaml b/.github/workflows/maven-publish.yaml index 7947f0c2..a3cf4c4d 100644 --- a/.github/workflows/maven-publish.yaml +++ b/.github/workflows/maven-publish.yaml @@ -8,13 +8,14 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 1.8 + java-version: 8 + distribution: 'zulu' - name: Cache Maven packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} diff --git a/Dockerfile-4_0 b/Dockerfile-4_0 index ab053f2f..3a177181 100644 --- a/Dockerfile-4_0 +++ b/Dockerfile-4_0 @@ -2,7 +2,7 @@ ARG CASSANDRA_VERSION=4.0.6 FROM --platform=$BUILDPLATFORM maven:3.6.3-jdk-8-slim as builder -ARG METRICS_COLLECTOR_VERSION=0.3.0 +ARG METRICS_COLLECTOR_VERSION=0.3.3 ARG CDC_AGENT_VERSION=2.0.0 ARG CDC_AGENT_EDITION=agent-c4 @@ -12,6 +12,7 @@ COPY pom.xml ./ COPY management-api-agent-common/pom.xml ./management-api-agent-common/pom.xml COPY management-api-agent-3.x/pom.xml ./management-api-agent-3.x/pom.xml COPY management-api-agent-4.x/pom.xml ./management-api-agent-4.x/pom.xml +COPY management-api-agent-4.1.x/pom.xml ./management-api-agent-4.1.x/pom.xml COPY management-api-common/pom.xml ./management-api-common/pom.xml COPY management-api-server/pom.xml ./management-api-server/pom.xml # this duplicates work done in the next steps, but this should provide @@ -21,6 +22,7 @@ RUN mvn -q -ff -T 1C install -DskipOpenApi && rm -rf target COPY management-api-agent-common ./management-api-agent-common COPY management-api-agent-3.x ./management-api-agent-3.x COPY management-api-agent-4.x ./management-api-agent-4.x +COPY management-api-agent-4.1.x ./management-api-agent-4.1.x COPY management-api-common ./management-api-common COPY management-api-server ./management-api-server RUN mvn -q -ff package -DskipTests diff --git a/Dockerfile-4_1 b/Dockerfile-4_1 new file mode 100644 index 00000000..4c29606f --- /dev/null +++ b/Dockerfile-4_1 @@ -0,0 +1,96 @@ +ARG CASSANDRA_VERSION=4.1-beta1 + +FROM --platform=$BUILDPLATFORM maven:3.6.3-jdk-8-slim as builder + +ARG METRICS_COLLECTOR_VERSION=0.3.3 +ARG METRICS_COLLECTOR_BUNDLE=${METRICS_COLLECTOR_VERSION}-4.1-beta1 +ARG CDC_AGENT_VERSION=2.0.0 +ARG CDC_AGENT_EDITION=agent-c4 + +WORKDIR /build + +COPY pom.xml ./ +COPY management-api-agent-common/pom.xml ./management-api-agent-common/pom.xml +COPY management-api-agent-3.x/pom.xml ./management-api-agent-3.x/pom.xml +COPY management-api-agent-4.x/pom.xml ./management-api-agent-4.x/pom.xml +COPY management-api-agent-4.1.x/pom.xml ./management-api-agent-4.1.x/pom.xml +COPY management-api-common/pom.xml ./management-api-common/pom.xml +COPY management-api-server/pom.xml ./management-api-server/pom.xml +# this duplicates work done in the next steps, but this should provide +# a solid cache layer that only gets reset on pom.xml changes +RUN mvn -q -ff -T 1C install -DskipOpenApi && rm -rf target + +COPY management-api-agent-common ./management-api-agent-common +COPY management-api-agent-3.x ./management-api-agent-3.x +COPY management-api-agent-4.x ./management-api-agent-4.x +COPY management-api-agent-4.1.x ./management-api-agent-4.1.x +COPY management-api-common ./management-api-common +COPY management-api-server ./management-api-server +RUN mvn -q -ff package -DskipTests + +# Download and extract Metrics Collector +ENV MCAC_PATH /opt/metrics-collector +RUN mkdir ${MCAC_PATH} && \ + if test ! -e datastax-mcac-agent-${METRICS_COLLECTOR_BUNDLE}.tar.gz; then curl -L -O "https://github.com/datastax/metric-collector-for-apache-cassandra/releases/download/v${METRICS_COLLECTOR_VERSION}/datastax-mcac-agent-${METRICS_COLLECTOR_BUNDLE}.tar.gz"; fi && \ + tar --directory ${MCAC_PATH} --strip-components 1 --gzip --extract --file datastax-mcac-agent-${METRICS_COLLECTOR_BUNDLE}.tar.gz + +ENV USER_HOME_PATH /home/cassandra +RUN mkdir ${USER_HOME_PATH} +# Download CDC agent +ENV CDC_AGENT_PATH=/opt/cdc_agent +RUN mkdir ${CDC_AGENT_PATH} && \ + curl -L -O "https://github.com/datastax/cdc-apache-cassandra/releases/download/v${CDC_AGENT_VERSION}/${CDC_AGENT_EDITION}-${CDC_AGENT_VERSION}-all.jar" && \ + mv ${CDC_AGENT_EDITION}-${CDC_AGENT_VERSION}-all.jar ${CDC_AGENT_PATH}/cdc-agent.jar + +FROM cassandra:${CASSANDRA_VERSION} as oss41 + +ARG TARGETARCH + +ENV CASSANDRA_PATH /opt/cassandra +ENV MAAC_PATH /opt/management-api +ENV MCAC_PATH /opt/metrics-collector +ENV CDC_AGENT_PATH=/opt/cdc_agent +ENV USER_HOME_PATH /home/cassandra + +ENV CASSANDRA_HOME ${CASSANDRA_PATH} +ENV CASSANDRA_CONF ${CASSANDRA_PATH}/conf +ENV CASSANDRA_LOGS ${CASSANDRA_PATH}/logs +# Log directory for Management API startup logs to avoid issues: +# https://datastax.jira.com/browse/DB-4627 +# https://issues.apache.org/jira/browse/CASSANDRA-16027 +ENV MGMT_API_LOG_DIR /var/log/cassandra + +COPY --from=builder --chown=cassandra:root /build/management-api-agent-4.1.x/target/datastax-mgmtapi-agent-4.1.x-0.1.0-SNAPSHOT.jar ${MAAC_PATH}/datastax-mgmtapi-agent-0.1.0-SNAPSHOT.jar +COPY --from=builder --chown=cassandra:root /build/management-api-server/target/datastax-mgmtapi-server-0.1.0-SNAPSHOT.jar ${MAAC_PATH}/ +COPY --from=builder --chown=cassandra:root ${MCAC_PATH} ${MCAC_PATH} +COPY --from=builder --chown=cassandra:root ${USER_HOME_PATH} ${USER_HOME_PATH} +COPY --from=builder --chown=cassandra:root ${CDC_AGENT_PATH} ${CDC_AGENT_PATH} + +# Setup user and fixup permissions +RUN chown -R cassandra:root ${CASSANDRA_PATH} && chmod -R g+w ${CASSANDRA_PATH} + +ENV TINI_VERSION v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TARGETARCH} /tini +RUN chmod +x /tini + +RUN set -eux; \ + rm -fr /etc/apt/sources.list.d/*; \ + rm -rf /var/lib/apt/lists/*; \ + apt-get update; \ + apt-get install -y --no-install-recommends wget iproute2; \ + rm -rf /var/lib/apt/lists/* + +# backwards compat with upstream ENTRYPOINT +COPY scripts/docker-entrypoint.sh /usr/local/bin/ +RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ + ln -sf /usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh && \ +# fix for the missing mtab in the containerd + ln -sf /proc/mounts /etc/mtab + +EXPOSE 9103 +EXPOSE 8080 + +USER cassandra + +ENTRYPOINT ["/tini", "-g", "--", "/docker-entrypoint.sh"] +CMD ["mgmtapi"] diff --git a/Dockerfile-oss b/Dockerfile-oss index b38218b8..bb60f476 100644 --- a/Dockerfile-oss +++ b/Dockerfile-oss @@ -2,7 +2,7 @@ ARG CASSANDRA_VERSION=3.11.13 FROM --platform=$BUILDPLATFORM maven:3.6.3-jdk-8-slim as builder -ARG METRICS_COLLECTOR_VERSION=0.3.0 +ARG METRICS_COLLECTOR_VERSION=0.3.3 ARG CDC_AGENT_VERSION=2.0.0 ARG CDC_AGENT_EDITION=agent-c3 @@ -12,6 +12,7 @@ COPY pom.xml ./ COPY management-api-agent-common/pom.xml ./management-api-agent-common/pom.xml COPY management-api-agent-3.x/pom.xml ./management-api-agent-3.x/pom.xml COPY management-api-agent-4.x/pom.xml ./management-api-agent-4.x/pom.xml +COPY management-api-agent-4.1.x/pom.xml ./management-api-agent-4.1.x/pom.xml COPY management-api-common/pom.xml ./management-api-common/pom.xml COPY management-api-server/pom.xml ./management-api-server/pom.xml # this duplicates work done in the next steps, but this should provide @@ -21,6 +22,7 @@ RUN mvn -q -ff -T 1C install -DskipOpenApi && rm -rf target COPY management-api-agent-common ./management-api-agent-common COPY management-api-agent-3.x ./management-api-agent-3.x COPY management-api-agent-4.x ./management-api-agent-4.x +COPY management-api-agent-4.1.x ./management-api-agent-4.1.x COPY management-api-common ./management-api-common COPY management-api-server ./management-api-server RUN mvn -q -ff package -DskipTests diff --git a/dse-68/Dockerfile.jdk11 b/dse-68/Dockerfile.jdk11 index 6d658c4a..4847919c 100644 --- a/dse-68/Dockerfile.jdk11 +++ b/dse-68/Dockerfile.jdk11 @@ -83,6 +83,7 @@ COPY pom.xml /tmp/pom.xml COPY management-api-agent-common/pom.xml /tmp/management-api-agent-common/pom.xml COPY management-api-agent-3.x/pom.xml /tmp/management-api-agent-3.x/pom.xml COPY management-api-agent-4.x/pom.xml /tmp/management-api-agent-4.x/pom.xml +COPY management-api-agent-4.1.x/pom.xml /tmp/management-api-agent-4.1.x/pom.xml COPY management-api-agent-dse-6.8/pom.xml tmp/management-api-agent-dse-6.8/pom.xml COPY management-api-common/pom.xml /tmp/management-api-common/pom.xml COPY management-api-server/pom.xml /tmp/management-api-server/pom.xml @@ -94,6 +95,7 @@ RUN cd /tmp && mvn -q -ff -T 1C install -DskipOpenApi -P dse && rm -rf target COPY management-api-agent-common /tmp/management-api-agent-common COPY management-api-agent-3.x /tmp/management-api-agent-3.x COPY management-api-agent-4.x /tmp/management-api-agent-4.x +COPY management-api-agent-4.1.x /tmp/management-api-agent-4.1.x COPY management-api-agent-dse-6.8 /tmp/management-api-agent-dse-6.8 COPY management-api-common /tmp/management-api-common COPY management-api-server /tmp/management-api-server diff --git a/dse-68/Dockerfile.jdk8 b/dse-68/Dockerfile.jdk8 index e8928f59..5af6a0a3 100644 --- a/dse-68/Dockerfile.jdk8 +++ b/dse-68/Dockerfile.jdk8 @@ -83,6 +83,7 @@ COPY pom.xml /tmp/pom.xml COPY management-api-agent-common/pom.xml /tmp/management-api-agent-common/pom.xml COPY management-api-agent-3.x/pom.xml /tmp/management-api-agent-3.x/pom.xml COPY management-api-agent-4.x/pom.xml /tmp/management-api-agent-4.x/pom.xml +COPY management-api-agent-4.1.x/pom.xml /tmp/management-api-agent-4.1.x/pom.xml COPY management-api-agent-dse-6.8/pom.xml tmp/management-api-agent-dse-6.8/pom.xml COPY management-api-common/pom.xml /tmp/management-api-common/pom.xml COPY management-api-server/pom.xml /tmp/management-api-server/pom.xml @@ -94,6 +95,7 @@ RUN cd /tmp && mvn -q -ff -T 1C install -DskipOpenApi -P dse && rm -rf target COPY management-api-agent-common /tmp/management-api-agent-common COPY management-api-agent-3.x /tmp/management-api-agent-3.x COPY management-api-agent-4.x /tmp/management-api-agent-4.x +COPY management-api-agent-4.1.x /tmp/management-api-agent-4.1.x COPY management-api-agent-dse-6.8 /tmp/management-api-agent-dse-6.8 COPY management-api-common /tmp/management-api-common COPY management-api-server /tmp/management-api-server diff --git a/management-api-agent-3.x/pom.xml b/management-api-agent-3.x/pom.xml index 56c62a69..9d9e5d9b 100644 --- a/management-api-agent-3.x/pom.xml +++ b/management-api-agent-3.x/pom.xml @@ -11,13 +11,6 @@ com.datastax datastax-mgmtapi-agent-3.x - - 1.10.10 - 3.1.5 - build_version.sh - 4.13.1 - - default diff --git a/management-api-agent-4.1.x/pom.xml b/management-api-agent-4.1.x/pom.xml new file mode 100644 index 00000000..eb315217 --- /dev/null +++ b/management-api-agent-4.1.x/pom.xml @@ -0,0 +1,189 @@ + + + 4.0.0 + + + com.datastax + datastax-mgmtapi + ${revision} + + + com.datastax + datastax-mgmtapi-agent-4.1.x + + + 4.1-beta1 + + + + + com.datastax + datastax-mgmtapi-common + ${project.version} + + + com.datastax + datastax-mgmtapi-agent-common + ${project.version} + + + net.bytebuddy + byte-buddy + ${bytebuddy.version} + + + net.bytebuddy + byte-buddy-agent + ${bytebuddy.version} + + + junit + junit + ${junit.version} + test + + + + + + + default + + true + + + + org.apache.cassandra + cassandra-all + ${cassandra4.version} + + + commons-codec + * + + + provided + + + + + dse-db-all + + false + + + + com.datastax.dse + dse-db-all + ${cassandra4.version} + + + commons-codec + * + + + provided + + + + + + + + + ${basedir}/src/main/resources + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.8 + + + initialize + parse-version + + parse-version + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.4.0 + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M3 + + + ${basedir} + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + UTF-8 + + + + org.apache.maven.plugins + maven-resources-plugin + 2.5 + + + UTF-8 + + + + org.apache.maven.plugins + maven-shade-plugin + 1.6 + + true + + + *:* + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + package + + shade + + + + + + + com.datastax.mgmtapi.Agent + + + + + + + + + + diff --git a/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/CassandraAPIServiceProvider41x.java b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/CassandraAPIServiceProvider41x.java new file mode 100644 index 00000000..63fb9d1a --- /dev/null +++ b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/CassandraAPIServiceProvider41x.java @@ -0,0 +1,18 @@ +/** + * Copyright DataStax, Inc. + * + * Please see the included license file for details. + */ +package com.datastax.mgmtapi; + +import com.datastax.mgmtapi.shim.CassandraAPI41x; +import com.datastax.mgmtapi.shims.CassandraAPI; + +public class CassandraAPIServiceProvider41x implements CassandraAPIServiceProvider { + + @Override + public CassandraAPI getCassandraAPI() { + return new CassandraAPI41x(); + } + +} diff --git a/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/GenericSerializer41x.java b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/GenericSerializer41x.java new file mode 100644 index 00000000..1f8864eb --- /dev/null +++ b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/GenericSerializer41x.java @@ -0,0 +1,148 @@ +/** + * Copyright DataStax, Inc. + * + * Please see the included license file for details. + */ +package com.datastax.mgmtapi.rpc; + +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.BooleanType; +import org.apache.cassandra.db.marshal.ByteType; +import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.marshal.DateType; +import org.apache.cassandra.db.marshal.DoubleType; +import org.apache.cassandra.db.marshal.EmptyType; +import org.apache.cassandra.db.marshal.FloatType; +import org.apache.cassandra.db.marshal.InetAddressType; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.ListType; +import org.apache.cassandra.db.marshal.LongType; +import org.apache.cassandra.db.marshal.MapType; +import org.apache.cassandra.db.marshal.SetType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.marshal.UUIDType; +import org.apache.cassandra.serializers.TypeSerializer; + +/** + * Uses reflection to look up an appropriate TypeSerializer/AbstractType to serialize objects + * without writing annoying ByteBufferUtil.bytes(X/Y/Z) boilerplate. + */ +class GenericSerializer41x +{ + // I considered using the drivers code (CodecRegistry, TypeCodec, etc) but decided that it made more sense to + // use the server side stuff from Cassandra. + + // extending this (at least for the purpose of RPC calls) is relatively straightforward: write a class that + // extends C*'s TypeSerializer and add to the map. For actually getting data into C*'s UDTs it might be a bit + // trickier. Unfortunately, there is not always a direct 1:1 mapping between Java and Cassandra types. A simple + // example is millisecond timestamps, which could be serialized as 'long' and 'timestamp'. The driver code + // actually has some bounds for this, but I think for us it will be simpler to just write more serializers and + // add them to the map. + private static final ConcurrentHashMap typeMap = new ConcurrentHashMap() {{ + put("void", EmptyType.instance); + put("boolean", BooleanType.instance); + put("java.lang.Boolean", BooleanType.instance); + put("byte", ByteType.instance); + put("java.lang.Byte", ByteType.instance); + put("int", Int32Type.instance); + put("java.lang.Integer", Int32Type.instance); + put("long", LongType.instance); + put("java.lang.Long", LongType.instance); + put("float", FloatType.instance); + put("java.lang.Float", FloatType.instance); + put("double", DoubleType.instance); + put("java.lang.Double", DoubleType.instance); + put("java.lang.String", UTF8Type.instance); + put("java.net.InetAddress", InetAddressType.instance); + put("java.util.Date", DateType.instance); + put("java.nio.ByteBuffer", BytesType.instance); + put("java.util.UUID", UUIDType.instance); + }}; + + static void registerType(String className, AbstractType type) + { + if (typeMap.putIfAbsent(className, type) != null) + { + throw new IllegalStateException("The type " + className + " is already registered."); + } + } + + static TypeSerializer getSerializer(Type type) + { + return getTypeOrException(type).getSerializer(); + } + + static AbstractType getTypeOrException(Type type) + { + AbstractType ctype = getType(type); + + if (ctype == null) + { + throw new AssertionError(String.format("Add type '%s' to GenericSerializer", type.getTypeName())); + } + + return ctype; + } + + static boolean simpleType(Type type) + { + return getType(type) != null; + } + + /** + * Most of the actual work is done here. Note that Generic type information is mostly destroyed at runtime + * (a list is just a list). For the Parameterized types to work correctly you have to call + * Method.getGenericParameterTypes() or something similar. Also, we currently punt on the frozen keyword. + * @return The C* abstract type corresponding to the Java type, or null if not found/impossible. + */ + static AbstractType getType(Type type) + { + assert type != null; + String strType = type.getTypeName(); + + // Rather than hard coding List List List etc we create them as needed. Also there + // is no need for a lock as the actual serializers do that for us. + if (!typeMap.containsKey(strType)) + { + if (type instanceof ParameterizedType) + { + ParameterizedType ptype = (ParameterizedType) type; + + if (ptype.getRawType().getTypeName().equals("java.util.List")) + { + assert ptype.getActualTypeArguments().length == 1; + typeMap.putIfAbsent(strType, + ListType.getInstance(getType(ptype.getActualTypeArguments()[0]), false)); + } + else if (ptype.getRawType().getTypeName().equals("java.util.Set")) + { + assert ptype.getActualTypeArguments().length == 1; + typeMap.putIfAbsent(strType, + SetType.getInstance(getType(ptype.getActualTypeArguments()[0]), false)); + } + else if (ptype.getRawType().getTypeName().equals("java.util.Map")) + { + assert ptype.getActualTypeArguments().length == 2; + typeMap.putIfAbsent(strType, + MapType.getInstance(getType(ptype.getActualTypeArguments()[0]), + getType(ptype.getActualTypeArguments()[1]), false)); + } + else + { + throw new AssertionError("Don't know how to serialize generic type '" + + ptype.getRawType().getTypeName() + "'"); + } + } + else + { + return null; + } + } + + return typeMap.get(strType); + } +} diff --git a/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/ObjectSerializer41x.java b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/ObjectSerializer41x.java new file mode 100644 index 00000000..f4485e0f --- /dev/null +++ b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/ObjectSerializer41x.java @@ -0,0 +1,137 @@ +/** + * Copyright DataStax, Inc. + * + * Please see the included license file for details. + */ +package com.datastax.mgmtapi.rpc; + +import java.lang.reflect.Field; +import java.lang.reflect.Type; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.collect.Lists; + +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.ColumnSpecification; +import org.apache.cassandra.cql3.ResultSet; +import org.apache.cassandra.cql3.ResultSet.ResultMetadata; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.TupleType; + +public class ObjectSerializer41x implements ObjectSerializer +{ + public final ImmutableSortedMap serializers; + + public class FieldSerializer + { + public final AbstractType type; + public final Function accessor; + + FieldSerializer(AbstractType type, Function accessor) + { + this.type = type; + this.accessor = accessor; + } + + FieldSerializer(AbstractType type, final Field field) + { + field.setAccessible(true); + this.type = type; + this.accessor = (obj) -> { + try + { + return field.get(obj); + } + catch (IllegalAccessException e) + { + throw new AssertionError("Should not happen as we set the field to accessible."); + } + }; + } + + ByteBuffer serializeField(T obj) + { + Object value = accessor.apply(obj); + if (value == null) + { + return null; + } + return type.getSerializer().serialize(accessor.apply(obj)); + } + } + + /** + * Due to the magic of java generics, the class doesn't have the full generic information, hence the double types. + * Also, this will only serialize **PUBLIC** fields (perhaps this should be changed; it's not totally clear). + * Tag accordingly. + */ + public ObjectSerializer41x(Class clazz, Type genericType) + { + serializers = GenericSerializer41x.simpleType(genericType) ? + ImmutableSortedMap.of("result", new FieldSerializer(GenericSerializer41x.getType(genericType), x -> x)) : + ImmutableSortedMap.copyOf(Arrays.stream(clazz.getFields()) + .collect(Collectors.toMap(field -> field.getName(), + field -> new FieldSerializer(GenericSerializer41x.getType(field.getType()), field)))); + // currently not recursive; multiple ways to do it + } + + public ObjectSerializer41x(Class clazz) + { + this(clazz, clazz); + } + + /** + * Serialize an object into a C* ResultSet, with each field as a named value. + * @param obj The object to serialize + * @param ksName Pretend we are coming from this keyspace + * @param cfName Pretend we are coming from this columnfamily + */ + public ResultSet toResultSet(T obj, String ksName, String cfName) + { + return new ResultSet( + new ResultMetadata(serializers.entrySet().stream() + .map(e -> new ColumnSpecification(ksName, cfName, + new ColumnIdentifier(e.getKey(), true), + e.getValue().type)) + .collect(Collectors.toList())), + Lists.>newArrayList(toByteBufferList(obj))); + } + + /** + * Serialize an object into a C* multi-row ResultSet, with each field as a named value. + * + * @param obj The object to serialize + * @param ksName Pretend we are coming from this keyspace + * @param cfName Pretend we are coming from this columnfamily + */ + public ResultSet toMultiRowResultSet(Collection obj, String ksName, String cfName) + { + return new ResultSet( + new ResultMetadata(serializers.entrySet().stream() + .map(e -> new ColumnSpecification(ksName, cfName, + new ColumnIdentifier(e.getKey(), true), + e.getValue().type)) + .collect(Collectors.toList())), + obj.stream().map(this::toByteBufferList).collect(Collectors.toList())); + } + + public List toByteBufferList(T obj) + { + return serializers.values().stream() + .map(fs -> fs.serializeField(obj)) + .collect(Collectors.toList()); + } + + public ByteBuffer toByteBuffer(T obj) + { + return TupleType.buildValue(serializers.values().stream() + .map(fs -> fs.serializeField(obj)) + .toArray(ByteBuffer[]::new)); + } +} diff --git a/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/RpcMethod41x.java b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/RpcMethod41x.java new file mode 100644 index 00000000..ab25bd5f --- /dev/null +++ b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/RpcMethod41x.java @@ -0,0 +1,195 @@ +/** + * Copyright DataStax, Inc. + * + * Please see the included license file for details. + */ +package com.datastax.mgmtapi.rpc; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.OptionalInt; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import com.google.common.base.Preconditions; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.cql3.ColumnIdentifier; +import org.apache.cassandra.cql3.ColumnSpecification; +import org.apache.cassandra.cql3.ResultSet; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.exceptions.RequestExecutionException; +import org.apache.cassandra.serializers.TypeSerializer; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.transport.messages.ResultMessage; + + +public class RpcMethod41x implements RpcMethod +{ + private static final Logger logger = LoggerFactory.getLogger(RpcMethod41x.class); + private final Method method; + private final RpcObject rpcObject; + private final String name; + private final List argSerializers; + private final List argTypes; + private final List argNames; + private final ObjectSerializer41x retSerializer; + private final OptionalInt clientStateArgIdx; + private final List> params; + + RpcMethod41x(Method method, RpcObject rpcObject) + { + this.method = method; + this.rpcObject = rpcObject; + this.name = method.getAnnotation(Rpc.class).name(); + + Annotation[][] allAnnotations = method.getParameterAnnotations(); + params = IntStream.range(0, method.getParameterCount()).boxed() + .flatMap(argIdx -> Arrays.stream(allAnnotations[argIdx]) + .filter(a -> a instanceof RpcParam) + .findFirst() + .map(RpcParam.class::cast) + .map(rpcParam -> Stream.of(Pair.of(argIdx, rpcParam))) + .orElseGet(Stream::empty)) + .collect(Collectors.toList()); + + Class[] paramTypes = method.getParameterTypes(); + clientStateArgIdx = IntStream.range(0, method.getParameterCount()) + .filter(argIdx -> paramTypes[argIdx] == RpcClientState.class) + .findFirst(); + + int expectedParamsCount = params.size() + (clientStateArgIdx.isPresent() ? 1 : 0); + if (method.getParameterCount() != expectedParamsCount) + { + throw new AssertionError(String.format( + "All arguments for %s.%s must be annotated with either RpcParam or RpcClientState", + rpcObject.getName(), + name)); + } + + Type[] genericParamTypes = method.getGenericParameterTypes(); + this.argSerializers = params.stream() + .map(p -> GenericSerializer41x.getSerializer(genericParamTypes[p.getKey()])) + .collect(Collectors.toList()); + + this.argTypes = params.stream() + .map(p -> GenericSerializer41x.getTypeOrException(genericParamTypes[p.getKey()])) + .collect(Collectors.toList()); + + this.argNames = params.stream() + .map(p -> p.getValue().name()) + .collect(Collectors.toList()); + + if (method.getAnnotation(Rpc.class).multiRow()) + { + Preconditions.checkArgument(Collection.class.isAssignableFrom(method.getReturnType()), + "If mutli-row result set is requested, the method return type must be an implementation of java.util.Collection"); + Type elemType = ((ParameterizedType) method.getGenericReturnType()).getActualTypeArguments()[0]; + Preconditions.checkArgument(elemType instanceof Class, + "If multi-row result set is request, the element type must be a Class"); + this.retSerializer = new ObjectSerializer41x<>((Class) elemType); + } + else + { + this.retSerializer = new ObjectSerializer41x<>(method.getReturnType(), method.getGenericReturnType()); + } + } + + public String getName() + { + return name; + } + + public int getArgumentCount() + { + return argTypes.size(); + } + + public ColumnSpecification getArgumentSpecification(int position) + { + return new ColumnSpecification("system", rpcObject.getName()+"."+name, new ColumnIdentifier(argNames.get(position), false), argTypes.get(position)); + } + + public ResultMessage execute(ClientState clientState, List parameters) + throws RequestExecutionException + { + try + { + RpcClientState rpcClientState = RpcClientState.fromClientState(clientState); + LazyRef rpcArgs = LazyRef.of(() -> getMethodArgs(rpcClientState, parameters)); + + // endpoint is not explicitly provided or points to this node -> execute locally + return toResultMessage(method.invoke(rpcObject.raw, rpcArgs.get())); + } + catch (Exception e) + { + throw createRpcExecutionException(e); + } + } + + private RpcExecutionException createRpcExecutionException(Throwable e) + { + String msg = String.format("Failed to execute method %s.%s", rpcObject.getName(), name); + logger.info(msg, e); + return RpcExecutionException.create(msg, e); + } + + private Object[] getMethodArgs(RpcClientState rpcClientState, Collection parameters) + { + Object[] args = new Object[method.getParameterCount()]; + clientStateArgIdx.ifPresent(idx -> args[idx] = rpcClientState); + Object[] rpcParams = deserializeParameters(parameters); + for (int i = 0; i < rpcParams.length; i++) + { + args[params.get(i).getKey()] = rpcParams[i]; + } + return args; + } + + public ResultSet toResultSet(Object object) + { + if (method.getAnnotation(Rpc.class).multiRow()) + { + return retSerializer.toMultiRowResultSet((Collection) object, rpcObject.getName(), name); + } + else + { + return retSerializer.toResultSet(object, rpcObject.getName(), name); + } + } + + public ResultMessage toResultMessage(Object object) + { + if (object == null) + { + return new ResultMessage.Void(); + } + else + { + return new ResultMessage.Rows(toResultSet(object)); + } + } + + private Object[] deserializeParameters(Collection args) + { + Object[] deserialized = new Object[args.size()]; + + int i = 0; + for (ByteBuffer arg : args) + { + deserialized[i] = arg != null ? argSerializers.get(i).deserialize(arg) : null; + i++; + } + + return deserialized; + } +} diff --git a/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/RpcMethodServiceProvider41x.java b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/RpcMethodServiceProvider41x.java new file mode 100644 index 00000000..01fe0c8c --- /dev/null +++ b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/rpc/RpcMethodServiceProvider41x.java @@ -0,0 +1,17 @@ +/** + * Copyright DataStax, Inc. + * + * Please see the included license file for details. + */ +package com.datastax.mgmtapi.rpc; + +import java.lang.reflect.Method; + +public class RpcMethodServiceProvider41x implements RpcMethodServiceProvider { + + @Override + public RpcMethod getRpcMethod(Method method, RpcObject rpcObject) { + return new RpcMethod41x(method, rpcObject); + } + +} diff --git a/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/shim/CassandraAPI41x.java b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/shim/CassandraAPI41x.java new file mode 100644 index 00000000..25bbc717 --- /dev/null +++ b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/shim/CassandraAPI41x.java @@ -0,0 +1,342 @@ +/** + * Copyright DataStax, Inc. + * + * Please see the included license file for details. + */ +package com.datastax.mgmtapi.shim; + +import java.lang.reflect.Field; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.datastax.mgmtapi.shims.CassandraAPI; +import com.datastax.mgmtapi.shims.RpcStatementShim; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import org.apache.cassandra.auth.IRoleManager; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.fql.FullQueryLoggerOptions; +import org.apache.cassandra.gms.ApplicationState; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.gms.VersionedValue; +import org.apache.cassandra.hints.HintsService; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.EndpointsForRange; +import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.K8SeedProvider41x; +import org.apache.cassandra.locator.ReplicaPlans; +import org.apache.cassandra.locator.SeedProvider; +import org.apache.cassandra.locator.TokenMetadata; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.streaming.SessionInfo; +import org.apache.cassandra.streaming.StreamManager; +import org.apache.cassandra.streaming.StreamState; +import org.apache.cassandra.streaming.management.StreamStateCompositeData; +import org.apache.cassandra.transport.Server; +import org.apache.cassandra.transport.UnixSocketServer41x; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.JVMStabilityInspector; + +public class CassandraAPI41x implements CassandraAPI +{ + private static final Logger logger = LoggerFactory.getLogger(CassandraAPI41x.class); + + private static final Supplier seedProvider = Suppliers.memoize(() -> new K8SeedProvider41x()); + + @Override + public void enableFullQuerylog() + { + logger.debug("Getting FQL options and calling enableFullQueryLogger."); + FullQueryLoggerOptions fqlOpts = DatabaseDescriptor.getFullQueryLogOptions(); + StorageService.instance.enableFullQueryLogger(fqlOpts.log_dir, + fqlOpts.roll_cycle, + fqlOpts.block, + fqlOpts.max_queue_weight, + fqlOpts.max_log_size, + fqlOpts.archive_command, + fqlOpts.max_archive_retries); + } + + @Override + public void disableFullQuerylog() + { + logger.debug("Stopping FullQueryLogger."); + StorageService.instance.stopFullQueryLogger(); + } + + @Override + public boolean isFullQueryLogEnabled() + { + boolean isEnabled = StorageService.instance.isFullQueryLogEnabled(); + logger.debug("Querying whether full query logging is enabled. Result is {}", isEnabled); + return isEnabled; + + } + + @Override + public void decommission(boolean force) throws InterruptedException + { + StorageService.instance.decommission(force); + } + + @Override + public Map, List> checkConsistencyLevel(String consistencyLevelName, Integer rfPerDc) + { + try + { + IPartitioner partitioner = DatabaseDescriptor.getPartitioner(); + IEndpointSnitch endpointSnitch = DatabaseDescriptor.getEndpointSnitch(); + TokenMetadata tokenMetadata = StorageService.instance.getTokenMetadata().cloneOnlyTokenMap(); + + ConsistencyLevel cl = ConsistencyLevel.valueOf(consistencyLevelName); + + Map dcNames = new HashMap<>(); + + for (InetAddressAndPort endpoint : tokenMetadata.getNormalAndBootstrappingTokenToEndpointMap().values()) + { + String dc = endpointSnitch.getDatacenter(endpoint); + assert dc != null; + + dcNames.put(dc, String.valueOf(rfPerDc)); + } + + Keyspace mockKs = Keyspace.mockKS(KeyspaceMetadata.create("none", KeyspaceParams.create(true, + ImmutableMap.builder().put("class", "NetworkTopologyStrategy").putAll(dcNames).build()))); + + AbstractReplicationStrategy mockStrategy = mockKs.getReplicationStrategy(); + mockStrategy.validateOptions(); + + Collection> tokenRanges = tokenMetadata.getPrimaryRangesFor(tokenMetadata.sortedTokens()); + + Map, List> results = new HashMap<>(); + + // For each range check the endpoints can achieve cl using the midpoint + for (Range range : tokenRanges) + { + Token midpoint = partitioner.midpoint(range.left, range.right); + EndpointsForRange endpoints = mockStrategy.calculateNaturalReplicas(midpoint, tokenMetadata); + + if (!ReplicaPlans.isSufficientLiveReplicasForRead(mockKs.getReplicationStrategy(), cl, endpoints)) + { + List downEndpoints = new ArrayList<>(); + for (InetAddressAndPort endpoint : endpoints.endpoints()) + { + EndpointState epState = Gossiper.instance.getEndpointStateForEndpoint(endpoint); + + if (!epState.isAlive()) + downEndpoints.add(endpoint.toString()); + } + + int blockFor = cl.blockFor(mockKs.getReplicationStrategy()); + + if (downEndpoints.isEmpty() && endpoints.size() < blockFor) + downEndpoints.add(String.format("%d replicas required, but only %d nodes in the ring", blockFor, endpoints.size())); + else if (downEndpoints.isEmpty()) + downEndpoints.add("Nodes Flapping"); + + results.put(ImmutableList.of((long) range.left.getTokenValue(), (long) range.right.getTokenValue()), downEndpoints); + } + } + return results; + } + catch (Throwable e) + { + logger.error("Exception encountered", e); + throw e; + } + } + + @Override + public SeedProvider getK8SeedProvider() + { + return seedProvider.get(); + } + + public Set reloadSeeds() + { + Field seedField = FBUtilities.getProtectedField(Gossiper.class, "seeds"); + + Set seeds = null; + try + { + seeds = (Set) seedField.get(Gossiper.instance); + } + catch (IllegalAccessException e) + { + throw new RuntimeException(e); + } + + // Get the new set in the same that buildSeedsList does + Set tmp = new HashSet<>(); + try + { + for (InetAddressAndPort seed : getK8SeedProvider().getSeeds()) + { + if (seed.equals(FBUtilities.getBroadcastAddressAndPort())) + continue; + tmp.add(seed); + } + } + // If using the SimpleSeedProvider invalid yaml added to the config since startup could + // cause this to throw. Additionally, third party seed providers may throw exceptions. + // Handle the error and return a null to indicate that there was a problem. + catch (Throwable e) + { + JVMStabilityInspector.inspectThrowable(e); + return null; + } + + if (tmp.size() == 0) + { + return seeds.stream().map(s -> s.getAddress()).collect(Collectors.toSet()); + } + + if (tmp.equals(seeds)) + { + return seeds.stream().map(s -> s.getAddress()).collect(Collectors.toSet()); + } + + // Add the new entries + seeds.addAll(tmp); + // Remove the old entries + seeds.retainAll(tmp); + logger.debug("New seed node list after reload {}", seeds); + + return seeds.stream().map(s -> s.getAddress()).collect(Collectors.toSet()); + } + + @Override + public ChannelInitializer makeSocketInitializer(Server.ConnectionTracker connectionTracker) + { + return UnixSocketServer41x.makeSocketInitializer(connectionTracker); + } + + @Override + public List> getEndpointStates() + { + List> result = new ArrayList<>(); + + for (InetAddressAndPort endpoint : Gossiper.instance.getEndpoints()) + { + EndpointState state = Gossiper.instance.getEndpointStateForEndpoint(endpoint); + Map states = new HashMap<>(); + for (Map.Entry s : state.states()) + { + states.put(s.getKey().name(), s.getValue().value); + } + + states.put("ENDPOINT_IP", endpoint.getHostAddress(false)); + states.put("IS_ALIVE", Boolean.toString(state.isAlive())); + result.add(states); + } + + return result; + } + + @Override + public List>>> getStreamInfo() + { + Set streams = StreamManager.instance.getCurrentStreams().stream() + .map(StreamStateCompositeData::fromCompositeData) + .collect(Collectors.toSet()); + + List>>> result = new ArrayList<>(); + + for (StreamState status : streams) + { + Map>> streamInfo = new HashMap<>(); + List> sessionResults = new ArrayList<>(); + + for (SessionInfo info : status.sessions) + { + Map sessionInfo = new HashMap<>(); + sessionInfo.put("STREAM_OPERATION", status.streamOperation.getDescription()); + sessionInfo.put("PEER", info.peer.toString()); + sessionInfo.put("USING_CONNECTION", info.connecting.toString()); + sessionInfo.put("TOTAL_FILES_TO_RECEIVE", String.valueOf(info.getTotalFilesToReceive())); + sessionInfo.put("TOTAL_FILES_RECEIVED", String.valueOf(info.getTotalFilesReceived())); + sessionInfo.put("TOTAL_SIZE_TO_RECEIVE", String.valueOf(info.getTotalSizeToReceive())); + sessionInfo.put("TOTAL_SIZE_RECEIVED", String.valueOf(info.getTotalSizeReceived())); + + sessionInfo.put("TOTAL_FILES_TO_SEND", String.valueOf(info.getTotalFilesToSend())); + sessionInfo.put("TOTAL_FILES_SENT", String.valueOf(info.getTotalFilesSent())); + sessionInfo.put("TOTAL_SIZE_TO_SEND", String.valueOf(info.getTotalSizeToSend())); + sessionInfo.put("TOTAL_SIZE_SENT", String.valueOf(info.getTotalSizeSent())); + sessionResults.add(sessionInfo); + } + + streamInfo.put(status.planId.toString(), sessionResults); + + result.add(streamInfo); + } + + return result; + } + + @Override + public StorageService getStorageService() + { + return StorageService.instance; + } + + @Override + public IRoleManager getRoleManager() + { + return DatabaseDescriptor.getRoleManager(); + } + + @Override + public CompactionManager getCompactionManager() + { + return CompactionManager.instance; + } + + @Override + public Gossiper getGossiper() + { + return Gossiper.instance; + } + + @Override + public String getLocalDataCenter() + { + return DatabaseDescriptor.getLocalDataCenter(); + } + + @Override + public RpcStatementShim makeRpcStatement(String method, String[] params) + { + return new RpcStatement41x(method, params); + } + + @Override + public HintsService getHintsService() + { + return HintsService.instance; + } +} diff --git a/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/shim/RpcStatement41x.java b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/shim/RpcStatement41x.java new file mode 100644 index 00000000..dcd6a067 --- /dev/null +++ b/management-api-agent-4.1.x/src/main/java/com/datastax/mgmtapi/shim/RpcStatement41x.java @@ -0,0 +1,62 @@ +package com.datastax.mgmtapi.shim; + +import com.datastax.mgmtapi.shims.RpcStatementShim; +import org.apache.cassandra.audit.AuditLogContext; +import org.apache.cassandra.cql3.QueryOptions; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.QueryState; +import org.apache.cassandra.transport.messages.ResultMessage; + +public class RpcStatement41x implements RpcStatementShim +{ + private final String method; + private final String[] params; + + public RpcStatement41x(String method, String[] params) + { + this.method = method; + this.params = params; + } + + @Override + public void authorize(ClientState clientState) + { + + } + + @Override + public void validate(ClientState clientState) + { + + } + + @Override + public ResultMessage execute(QueryState queryState, QueryOptions queryOptions, long l) + { + return new ResultMessage.Void(); + } + + @Override + public ResultMessage executeLocally(QueryState queryState, QueryOptions queryOptions) + { + return new ResultMessage.Void(); + } + + @Override + public AuditLogContext getAuditLogContext() + { + return null; + } + + @Override + public String getMethod() + { + return method; + } + + @Override + public String[] getParams() + { + return params; + } +} diff --git a/management-api-agent-4.1.x/src/main/java/org/apache/cassandra/locator/K8SeedProvider41x.java b/management-api-agent-4.1.x/src/main/java/org/apache/cassandra/locator/K8SeedProvider41x.java new file mode 100644 index 00000000..d72d633f --- /dev/null +++ b/management-api-agent-4.1.x/src/main/java/org/apache/cassandra/locator/K8SeedProvider41x.java @@ -0,0 +1,63 @@ +/** + * Copyright DataStax, Inc. + * + * Please see the included license file for details. + */ +package org.apache.cassandra.locator; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.config.DatabaseDescriptor; + +public class K8SeedProvider41x implements SeedProvider +{ + private static final Logger logger = LoggerFactory.getLogger(K8SeedProvider41x.class); + + public K8SeedProvider41x() { + } + + public List getSeeds() + { + Config conf; + try + { + conf = DatabaseDescriptor.loadConfig(); + } + catch (Exception e) + { + throw new AssertionError(e); + } + String[] hosts = conf.seed_provider.parameters.get("seeds").split(",", -1); + List seeds = new ArrayList<>(hosts.length); + for (String host : hosts) + { + try + { + // A name may resolve to multiple seed node IPs, as would be + // the case in Kubernetes when a headless service is used to + // represent the seed nodes in a cluster, which is why we use + // `getAllByName` here instead of `getByName`. + seeds.addAll(Arrays.asList(InetAddress.getAllByName(host.trim())) + .stream() + .map(n -> InetAddressAndPort.getByAddress(n)) + .collect(Collectors.toList())); + } + catch (UnknownHostException ex) + { + // not fatal... DD will bark if there end up being zero seeds. + logger.warn("Seed provider couldn't lookup host {}", host); + } + } + return Collections.unmodifiableList(seeds); + } +} diff --git a/management-api-agent-4.1.x/src/main/java/org/apache/cassandra/transport/UnixSocketServer41x.java b/management-api-agent-4.1.x/src/main/java/org/apache/cassandra/transport/UnixSocketServer41x.java new file mode 100644 index 00000000..3ff6785d --- /dev/null +++ b/management-api-agent-4.1.x/src/main/java/org/apache/cassandra/transport/UnixSocketServer41x.java @@ -0,0 +1,321 @@ +/** + * Copyright DataStax, Inc. + * + * Please see the included license file for details. + */ +package org.apache.cassandra.transport; + +import com.datastax.mgmtapi.ipc.IPCController; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.ChannelPromise; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.VoidChannelPromise; +import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.util.Attribute; +import org.apache.cassandra.auth.IAuthenticator; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.service.ClientState; +import org.apache.cassandra.service.ClientWarn; +import org.apache.cassandra.service.QueryState; +import org.apache.cassandra.transport.ClientResourceLimits.Overload; +import org.apache.cassandra.transport.messages.AuthenticateMessage; +import org.apache.cassandra.transport.messages.ErrorMessage; +import org.apache.cassandra.transport.messages.ReadyMessage; +import org.apache.cassandra.transport.messages.StartupMessage; +import org.apache.cassandra.transport.messages.SupportedMessage; +import org.apache.cassandra.utils.JVMStabilityInspector; + +public class UnixSocketServer41x +{ + private static final Logger logger = LoggerFactory.getLogger(IPCController.class); + + // Names of handlers used in pre-V5 pipelines + private static final String ENVELOPE_DECODER = "envelopeDecoder"; + private static final String ENVELOPE_ENCODER = "envelopeEncoder"; + private static final String MESSAGE_DECOMPRESSOR = "decompressor"; + private static final String MESSAGE_COMPRESSOR = "compressor"; + private static final String MESSAGE_DECODER = "messageDecoder"; + private static final String MESSAGE_ENCODER = "messageEncoder"; + private static final String LEGACY_MESSAGE_PROCESSOR = "legacyCqlProcessor"; + private static final String INITIAL_HANDLER = "initialHandler"; + private static final String EXCEPTION_HANDLER = "exceptionHandler"; + + public static ChannelInitializer makeSocketInitializer(final Server.ConnectionTracker connectionTracker) + { + logger.debug("Creating Channel Initializer"); + return new ChannelInitializer() + { + @Override + protected void initChannel(Channel channel) throws Exception + { + ChannelPipeline pipeline = channel.pipeline(); + + pipeline.addLast(ENVELOPE_ENCODER, Envelope.Encoder.instance); + pipeline.addLast(INITIAL_HANDLER, + new PipelineChannelInitializer( + new Envelope.Decoder(), + (channel1, version) -> + new UnixSocketConnection( + channel1, + version, + connectionTracker))); + // The exceptionHandler will take care of handling exceptionCaught(...) events while still running + // on the same EventLoop as all previous added handlers in the pipeline. This is important as the used + // eventExecutorGroup may not enforce strict ordering for channel events. + // As the exceptionHandler runs in the EventLoop as the previous handlers we are sure all exceptions are + // correctly handled before the handler itself is removed. + // See https://issues.apache.org/jira/browse/CASSANDRA-13649 + pipeline.addLast(EXCEPTION_HANDLER, PreV5Handlers.ExceptionHandler.instance); + } + }; + } + + @ChannelHandler.Sharable + static class UnixSockMessage extends SimpleChannelInboundHandler + { + @Override + protected void channelRead0(ChannelHandlerContext ctx, Message.Request request) throws Exception + { + final Message.Response response; + final UnixSocketConnection connection; + long queryStartNanoTime = System.nanoTime(); + + try + { + assert request.connection() instanceof UnixSocketConnection; + connection = (UnixSocketConnection) request.connection(); + if (connection.getVersion().isGreaterOrEqualTo(ProtocolVersion.V4)) + ClientWarn.instance.captureWarnings(); + + QueryState qstate = connection.validateNewMessage(request.type, connection.getVersion(), request.getStreamId()); + //logger.info("Executing {} {} {}", request, connection.getVersion(), request.getStreamId()); + + Message.Response r = request.execute(qstate, queryStartNanoTime); + + //UnixSocket has no auth + response = r instanceof AuthenticateMessage ? new ReadyMessage() : r; + + response.setStreamId(request.getStreamId()); + response.setWarnings(ClientWarn.instance.getWarnings()); + response.attach(connection); + connection.applyStateTransition(request.type, response.type); + } + catch (Throwable t) + { + //logger.warn("Exception encountered", t); + JVMStabilityInspector.inspectThrowable(t); + ExceptionHandlers.UnexpectedChannelExceptionHandler handler = new ExceptionHandlers.UnexpectedChannelExceptionHandler(ctx.channel(), true); + ctx.writeAndFlush(ErrorMessage.fromException(t, handler).setStreamId(request.getStreamId())); + request.getSource().release(); + return; + } + finally + { + ClientWarn.instance.resetWarnings(); + } + + ctx.writeAndFlush(response); + request.getSource().release(); + } + } + + static class UnixSocketConnection extends ServerConnection + { + private enum State { UNINITIALIZED, AUTHENTICATION, READY } + + private final ClientState clientState; + private volatile State state; + // private final ConcurrentMap queryStates = new ConcurrentHashMap<>(); + + public UnixSocketConnection(Channel channel, ProtocolVersion version, Connection.Tracker tracker) + { + super(channel, version, tracker); + this.clientState = ClientState.forInternalCalls(); + this.state = State.UNINITIALIZED; + } + + @Override + public QueryState validateNewMessage(Message.Type type, ProtocolVersion version) + { + return validateNewMessage(type, version, -1); + } + + public QueryState validateNewMessage(Message.Type type, ProtocolVersion version, int streamId) + { + switch (state) + { + case UNINITIALIZED: + if (type != Message.Type.STARTUP && type != Message.Type.OPTIONS) + throw new ProtocolException(String.format("Unexpected message %s, expecting STARTUP or OPTIONS", type)); + break; + case AUTHENTICATION: + // Support both SASL auth from protocol v2 and the older style Credentials auth from v1 + if (type != Message.Type.AUTH_RESPONSE && type != Message.Type.CREDENTIALS) + throw new ProtocolException(String.format("Unexpected message %s, expecting %s", type, version == ProtocolVersion.V1 ? "CREDENTIALS" : "SASL_RESPONSE")); + break; + case READY: + if (type == Message.Type.STARTUP) + throw new ProtocolException("Unexpected message STARTUP, the connection is already initialized"); + break; + default: + throw new AssertionError(); + } + return new QueryState(clientState); + } + + @Override + public void applyStateTransition(Message.Type requestType, Message.Type responseType) + { + switch (state) + { + case UNINITIALIZED: + if (requestType == Message.Type.STARTUP) + { + // Just set the state to READY as the Unix socket needs to bypass authentication + state = State.READY; + } + break; + case AUTHENTICATION: + // Support both SASL auth from protocol v2 and the older style Credentials auth from v1 + assert requestType == Message.Type.AUTH_RESPONSE || requestType == Message.Type.CREDENTIALS; + + if (responseType == Message.Type.READY || responseType == Message.Type.AUTH_SUCCESS) + { + state = State.READY; + // we won't use the authenticator again, null it so that it can be GC'd + } + break; + case READY: + break; + default: + throw new AssertionError(); + } + } + + @Override + public IAuthenticator.SaslNegotiator getSaslNegotiator(QueryState queryState) + { + return null; + } + } + + static class PipelineChannelInitializer extends ByteToMessageDecoder + { + Envelope.Decoder decoder; + Connection.Factory factory; + + PipelineChannelInitializer(Envelope.Decoder decoder, Connection.Factory factory) + { + this.decoder = decoder; + this.factory = factory; + } + + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List out) throws Exception { + Envelope inbound = decoder.decode(buffer); + if (inbound == null) + return; + + try + { + Envelope outbound; + switch (inbound.header.type) + { + case OPTIONS: + logger.debug("OPTIONS received {}", inbound.header.version); + List cqlVersions = new ArrayList<>(); + cqlVersions.add(QueryProcessor.CQL_VERSION.toString()); + + List compressions = new ArrayList<>(); + if (Compressor.SnappyCompressor.instance != null) + compressions.add("snappy"); + // LZ4 is always available since worst case scenario it default to a pure JAVA implem. + compressions.add("lz4"); + + Map> supportedOptions = new HashMap<>(); + supportedOptions.put(StartupMessage.CQL_VERSION, cqlVersions); + supportedOptions.put(StartupMessage.COMPRESSION, compressions); + supportedOptions.put(StartupMessage.PROTOCOL_VERSIONS, ProtocolVersion.supportedVersions()); + SupportedMessage supported = new SupportedMessage(supportedOptions); + outbound = supported.encode(inbound.header.version); + ctx.writeAndFlush(outbound); + break; + + case STARTUP: + Attribute attrConn = ctx.channel().attr(Connection.attributeKey); + Connection connection = attrConn.get(); + if (connection == null) + { + connection = factory.newConnection(ctx.channel(), inbound.header.version); + attrConn.set(connection); + } + assert connection instanceof ServerConnection; + + StartupMessage startup = (StartupMessage) Message.Decoder.decodeMessage(ctx.channel(), inbound); + // InetAddress remoteAddress = ((InetSocketAddress) ctx.channel().remoteAddress()).getAddress(); + // final ClientResourceLimits.Allocator allocator = ClientResourceLimits.getAllocatorForEndpoint(remoteAddress); + + ChannelPromise promise; + if (inbound.header.version.isGreaterOrEqualTo(ProtocolVersion.V5)) + { + // v5 not yet supported + logger.warn("PROTOCOL v5 not yet supported."); + + } + // no need to configure the pipeline asynchronously in this case + // the capacity obtained from allocator for the STARTUP message + // is released when flushed by the legacy dispatcher/flusher so + // there's no need to explicitly release that here either. + + ChannelPipeline pipeline = ctx.channel().pipeline(); + pipeline.addBefore(ENVELOPE_ENCODER, ENVELOPE_DECODER, new Envelope.Decoder()); + pipeline.addBefore(INITIAL_HANDLER, MESSAGE_DECOMPRESSOR, Envelope.Decompressor.instance); + pipeline.addBefore(INITIAL_HANDLER, MESSAGE_COMPRESSOR, Envelope.Compressor.instance); + pipeline.addBefore(INITIAL_HANDLER, MESSAGE_DECODER, PreV5Handlers.ProtocolDecoder.instance); + pipeline.addBefore(INITIAL_HANDLER, MESSAGE_ENCODER, PreV5Handlers.ProtocolEncoder.instance); + pipeline.addBefore(INITIAL_HANDLER, LEGACY_MESSAGE_PROCESSOR, new UnixSockMessage()); + pipeline.remove(INITIAL_HANDLER); + + + promise = new VoidChannelPromise(ctx.channel(), false); + + Message.Response response = Dispatcher.processRequest(ctx.channel(), startup, Overload.NONE); + + if (response.type.equals(Message.Type.AUTHENTICATE)) + // bypass authentication + response = new ReadyMessage(); + + outbound = response.encode(inbound.header.version); + ctx.writeAndFlush(outbound, promise); + logger.debug("Configured pipeline: {}", ctx.pipeline()); + break; + + default: + ErrorMessage error = + ErrorMessage.fromException( + new ProtocolException(String.format("Unexpected message %s, expecting STARTUP or OPTIONS", + inbound.header.type))); + outbound = error.encode(inbound.header.version); + ctx.writeAndFlush(outbound); + } + } + finally + { + inbound.release(); + } + } + } +} diff --git a/management-api-agent-4.1.x/src/main/resources/META-INF/services/com.datastax.mgmtapi.CassandraAPIServiceProvider b/management-api-agent-4.1.x/src/main/resources/META-INF/services/com.datastax.mgmtapi.CassandraAPIServiceProvider new file mode 100644 index 00000000..11cc0f85 --- /dev/null +++ b/management-api-agent-4.1.x/src/main/resources/META-INF/services/com.datastax.mgmtapi.CassandraAPIServiceProvider @@ -0,0 +1 @@ +com.datastax.mgmtapi.CassandraAPIServiceProvider41x diff --git a/management-api-agent-4.1.x/src/main/resources/META-INF/services/com.datastax.mgmtapi.rpc.RpcMethodServiceProvider b/management-api-agent-4.1.x/src/main/resources/META-INF/services/com.datastax.mgmtapi.rpc.RpcMethodServiceProvider new file mode 100644 index 00000000..416dc4a3 --- /dev/null +++ b/management-api-agent-4.1.x/src/main/resources/META-INF/services/com.datastax.mgmtapi.rpc.RpcMethodServiceProvider @@ -0,0 +1 @@ +com.datastax.mgmtapi.rpc.RpcMethodServiceProvider41x diff --git a/management-api-agent-4.x/pom.xml b/management-api-agent-4.x/pom.xml index 5bc746db..2284732b 100644 --- a/management-api-agent-4.x/pom.xml +++ b/management-api-agent-4.x/pom.xml @@ -11,13 +11,6 @@ com.datastax datastax-mgmtapi-agent-4.x - - 1.10.10 - 3.1.5 - build_version.sh - 4.13.1 - - com.datastax diff --git a/management-api-agent-common/pom.xml b/management-api-agent-common/pom.xml index ad4ff9a4..b6209dbc 100644 --- a/management-api-agent-common/pom.xml +++ b/management-api-agent-common/pom.xml @@ -11,13 +11,6 @@ com.datastax datastax-mgmtapi-agent-common - - 1.10.10 - 3.1.5 - build_version.sh - 4.13.1 - - default diff --git a/management-api-agent-common/src/main/java/com/datastax/mgmtapi/NodeOpsProvider.java b/management-api-agent-common/src/main/java/com/datastax/mgmtapi/NodeOpsProvider.java index ecdd8b99..36ecc11b 100644 --- a/management-api-agent-common/src/main/java/com/datastax/mgmtapi/NodeOpsProvider.java +++ b/management-api-agent-common/src/main/java/com/datastax/mgmtapi/NodeOpsProvider.java @@ -637,7 +637,8 @@ public List> getSnapshotDetails(@RpcParam(name="snapshotName Map detailsMap = new HashMap<>(); for (String itemName : compositeData.getCompositeType().keySet()) { - String value = compositeData.get(itemName).toString(); + Object item = compositeData.get(itemName); + String value = item == null ? "null" : item.toString(); detailsMap.put(itemName, value); } detailsList.add(detailsMap); diff --git a/management-api-agent-common/src/main/java/com/datastax/mgmtapi/rpc/RpcMethod.java b/management-api-agent-common/src/main/java/com/datastax/mgmtapi/rpc/RpcMethod.java index b6632a85..a3aa68d4 100644 --- a/management-api-agent-common/src/main/java/com/datastax/mgmtapi/rpc/RpcMethod.java +++ b/management-api-agent-common/src/main/java/com/datastax/mgmtapi/rpc/RpcMethod.java @@ -1,4 +1,4 @@ -/* +/** * Copyright DataStax, Inc. * * Please see the included license file for details. @@ -12,7 +12,7 @@ public interface RpcMethod { - String getName(); + String getName(); public int getArgumentCount(); diff --git a/management-api-agent-dse-6.8/pom.xml b/management-api-agent-dse-6.8/pom.xml index 7d03d8b1..c8208ab9 100644 --- a/management-api-agent-dse-6.8/pom.xml +++ b/management-api-agent-dse-6.8/pom.xml @@ -29,11 +29,6 @@ 6.8.26 - 1.9.15 - 3.1.5 - 4.10.0 - 4.13.1 - build_version.sh diff --git a/management-api-common/pom.xml b/management-api-common/pom.xml index 64f14d78..1a6bfb6f 100644 --- a/management-api-common/pom.xml +++ b/management-api-common/pom.xml @@ -11,14 +11,6 @@ com.datastax datastax-mgmtapi-common - - 1.7.25 - 1.2.9 - 4.1.50.Final - 4.13.1 - 3.5.13 - - org.slf4j diff --git a/management-api-server/pom.xml b/management-api-server/pom.xml index b6a4c89b..2474e812 100644 --- a/management-api-server/pom.xml +++ b/management-api-server/pom.xml @@ -12,19 +12,13 @@ datastax-mgmtapi-server - 1.7.25 - 1.2.9 2.1.1 30.1.1-jre 2.7.0 2.1.6 4.5.9.Final - 4.1.50.Final - 3.2.13 4.0.3 3.17.2 - 4.13.1 - 3.5.13 3.1.0 diff --git a/management-api-server/src/test/java/com/datastax/mgmtapi/BaseDockerIntegrationTest.java b/management-api-server/src/test/java/com/datastax/mgmtapi/BaseDockerIntegrationTest.java index 532e010f..13ed2cc8 100644 --- a/management-api-server/src/test/java/com/datastax/mgmtapi/BaseDockerIntegrationTest.java +++ b/management-api-server/src/test/java/com/datastax/mgmtapi/BaseDockerIntegrationTest.java @@ -105,12 +105,14 @@ protected void finished(Description description) @Parameterized.Parameters(name = "{index}: {0}") public static List testVersions() { - List versions = new ArrayList<>(3); + List versions = new ArrayList<>(4); if (Boolean.getBoolean("run311tests")) versions.add("3_11"); if (Boolean.getBoolean("run40tests")) versions.add("4_0"); + if (Boolean.getBoolean("run41tests")) + versions.add("4_1"); if (Boolean.getBoolean("runDSEtests")) versions.add("dse-68"); diff --git a/management-api-server/src/test/java/com/datastax/mgmtapi/NonDestructiveOpsIT.java b/management-api-server/src/test/java/com/datastax/mgmtapi/NonDestructiveOpsIT.java index 67e853da..4c69aeac 100644 --- a/management-api-server/src/test/java/com/datastax/mgmtapi/NonDestructiveOpsIT.java +++ b/management-api-server/src/test/java/com/datastax/mgmtapi/NonDestructiveOpsIT.java @@ -240,7 +240,7 @@ public void testTruncateWithoutHost() throws IOException, URISyntaxException @Test public void testResetLocalSchema() throws IOException, URISyntaxException { - assumeTrue(IntegrationTestUtils.shouldRun()); + assumeTrue(this.version != "4_1" && IntegrationTestUtils.shouldRun()); ensureStarted(); NettyHttpClient client = new NettyHttpClient(BASE_URL); @@ -300,7 +300,6 @@ public void testGetEndpoints() throws IOException, URISyntaxException return responseAsString(r); }).join(); - System.err.println(response); assertNotNull(response); assertNotEquals("", response); } diff --git a/management-api-server/src/test/java/com/datastax/mgmtapi/helpers/DockerHelper.java b/management-api-server/src/test/java/com/datastax/mgmtapi/helpers/DockerHelper.java index 8b5899d4..d7d11ea1 100644 --- a/management-api-server/src/test/java/com/datastax/mgmtapi/helpers/DockerHelper.java +++ b/management-api-server/src/test/java/com/datastax/mgmtapi/helpers/DockerHelper.java @@ -401,32 +401,29 @@ static DockerBuildConfig getConfig(String version, List envVars) config.dockerFile = Paths.get(config.baseDir.getPath(), "Dockerfile-oss").toFile(); config.target = "oss311"; config.envList = Lists.newArrayList("MAX_HEAP_SIZE=500M", "HEAP_NEWSIZE=100M"); - if (envVars != null) - { - config.envList.addAll(envVars); - } break; case "4_0" : config.dockerFile = Paths.get(config.baseDir.getPath(), "Dockerfile-4_0").toFile(); config.target = "oss40"; config.envList = Lists.newArrayList("MAX_HEAP_SIZE=500M", "HEAP_NEWSIZE=100M"); - if (envVars != null) - { - config.envList.addAll(envVars); - } + break; + case "4_1" : + config.dockerFile = Paths.get(config.baseDir.getPath(), "Dockerfile-4_1").toFile(); + config.target = "oss41"; + config.envList = Lists.newArrayList("MAX_HEAP_SIZE=500M", "HEAP_NEWSIZE=100M"); break; case "dse-68" : config.dockerFile = Paths.get(config.baseDir.getPath(), "dse-68", "Dockerfile.jdk11").toFile(); config.target = "dse68"; config.envList = Lists.newArrayList("MAX_HEAP_SIZE=500M", "HEAP_NEWSIZE=100M", "DS_LICENSE=accept", "USE_MGMT_API=true"); - if (envVars != null) - { - config.envList.addAll(envVars); - } break; default : throw new RuntimeException("Unsupported Cassandra version: " + version); } + if (envVars != null) + { + config.envList.addAll(envVars); + } return config; } } diff --git a/management-api-server/src/test/resources/com/datastax/mgmtapi/dcrf-override-1.yaml b/management-api-server/src/test/resources/com/datastax/mgmtapi/dcrf-override-1.yaml index 4ea15bb1..724d47ac 100644 --- a/management-api-server/src/test/resources/com/datastax/mgmtapi/dcrf-override-1.yaml +++ b/management-api-server/src/test/resources/com/datastax/mgmtapi/dcrf-override-1.yaml @@ -44,10 +44,10 @@ spec: authenticator: org.apache.cassandra.auth.PasswordAuthenticator authorizer: org.apache.cassandra.auth.CassandraAuthorizer auto_snapshot: false # default is true, but we don't have a good way to use or clean up snapshots when someone drops a table - compaction_throughput_mb_per_sec: 64 # default is 16 + #compaction_throughput_mb_per_sec: 64 # default is 16 concurrent_compactors: 2 # this is based on tuning for four cores / 15 GB file_cache_size_in_mb: 500 # this is based on tuning for four cores / 15 GB - hinted_handoff_throttle_in_kb: 512 # this is based on tuning for four cores / 15 GB + #hinted_handoff_throttle_in_kb: 512 # this is based on tuning for four cores / 15 GB memtable_flush_writers: 2 # this is based on tuning for four cores / 15 GB num_tokens: 256 # default is 1 phi_convict_threshold: 12 # default is 8 diff --git a/management-api-server/src/test/resources/com/datastax/mgmtapi/operator-sample.yaml b/management-api-server/src/test/resources/com/datastax/mgmtapi/operator-sample.yaml index 191aae94..725d6f49 100644 --- a/management-api-server/src/test/resources/com/datastax/mgmtapi/operator-sample.yaml +++ b/management-api-server/src/test/resources/com/datastax/mgmtapi/operator-sample.yaml @@ -44,10 +44,10 @@ spec: authenticator: org.apache.cassandra.auth.PasswordAuthenticator authorizer: org.apache.cassandra.auth.CassandraAuthorizer auto_snapshot: false # default is true, but we don't have a good way to use or clean up snapshots when someone drops a table - compaction_throughput_mb_per_sec: 64 # default is 16 + #compaction_throughput_mb_per_sec: 64 # default is 16 concurrent_compactors: 2 # this is based on tuning for four cores / 15 GB file_cache_size_in_mb: 500 # this is based on tuning for four cores / 15 GB - hinted_handoff_throttle_in_kb: 512 # this is based on tuning for four cores / 15 GB + #hinted_handoff_throttle_in_kb: 512 # this is based on tuning for four cores / 15 GB memtable_flush_writers: 2 # this is based on tuning for four cores / 15 GB num_tokens: 256 # default is 1 phi_convict_threshold: 12 # default is 8 diff --git a/pom.xml b/pom.xml index 6bedbfa9..b76a912e 100644 --- a/pom.xml +++ b/pom.xml @@ -11,9 +11,17 @@ build_version.sh 0.1.0-SNAPSHOT - 4.11.1 + 4.15.0 3.11.13 - 4.0.4 + 4.0.6 + 3.2.13 + 4.13.2 + 1.10.10 + build_version.sh + 1.7.25 + 1.2.9 + 4.1.50.Final + 3.5.13 @@ -33,6 +41,7 @@ management-api-agent-common management-api-agent-3.x management-api-agent-4.x + management-api-agent-4.1.x management-api-server @@ -49,6 +58,7 @@ management-api-agent-common management-api-agent-3.x management-api-agent-4.x + management-api-agent-4.1.x management-api-agent-dse-6.8 management-api-server