比较版本

results

jmh-benchmarks/generated

jmh-benchmarks/src/main/generated

streams/src/generated

.netrc

.semaphore-cache/

.ssh

vendor/

/tmp/

version: v1.0

name: ce-kafka

agent:

  machine:

    type: e1-standard-8

    os_image: ubuntu1804

blocks:

  - name: "Build, Test, Release"

    task:

      secrets:

        - name: vault_sem2_approle_prod

      prologue:

        commands:

          - checkout

          - make install-vault

          - . mk-include/bin/vault-setup

          - . vault-sem-get-secret semaphore-secrets-global

          - . vault-sem-get-secret artifactory-docker-helm

          - . vault-sem-get-secret testbreak-reporting

          - . vault-sem-get-secret aws_credentials

          - . vault-sem-get-secret ssh_id_rsa

          - . vault-sem-get-secret ssh_config

          - . vault-sem-get-secret gitconfig

          - . vault-sem-get-secret netrc

          - . vault-sem-get-secret maven-settings

          - . vault-sem-get-secret gradle_properties

          - chmod 400 ~/.ssh/id_rsa

          - make init-ci

          - sem-version java 8

          - sem-version go 1.12

          - git config --global url."git@github.com:".insteadOf "https://github.com/"

          - export SEMAPHORE_CACHE_DIR=/home/semaphore

          - source /home/semaphore/.testbreak/setup.sh

      jobs:

        - name: Setup, build, release

          commands:

            - make init-ci

            - make build

            - make test

            - make release-ci

      epilogue:

        commands:

          - source /home/semaphore/.testbreak/after.sh

 No newline at end of file

Copyright 2016 Confluent, Inc.

Confluent Open Source

==========================

This is a [Confluent](https://www.confluent.io/) Open Source fork of Apache Kafka.

This version includes several modifications to enhance maintainability and ease-of-use.

Just like Apache Kafka, COS is distributed under the Apache 2.0 license.

##########

##########

FROM openjdk:11-jdk as kafka-builder

USER root

COPY . /home/gradle

WORKDIR /home/gradle

# /root/.gradle is a docker volume so we can't copy files into it

ENV GRADLE_USER_HOME=/root/gradle-home

RUN mkdir -p /root/.m2/repository $GRADLE_USER_HOME \

  && cp ./tmp/gradle/gradle.properties $GRADLE_USER_HOME

RUN ./gradlew clean releaseTarGz -x signArchives --stacktrace -PpackageMetricsReporter=true && ./gradlew install --stacktrace

WORKDIR /build

# The build generates two tgz files, one with compiled code and one

# with site-docs. The pattern needs to include part of the version

# before .tgz so we only match the code jar and not the site-docs

# jar.

RUN tar -xzvf /home/gradle/core/build/distributions/kafka_*-ce.tgz --strip-components 1

##########

# Build a Docker image for the K8s liveness storage probe.

FROM golang:1.12.7 as go-build

ARG version

WORKDIR /root

COPY .ssh .ssh

COPY .netrc ./

RUN ssh-keyscan -t rsa github.com > /root/.ssh/known_hosts

WORKDIR /go/src/github.com/confluentinc/ce-kafka/cc-services/storage_probe

COPY cc-services/storage_probe .

COPY ./mk-include ./mk-include

RUN make deps DEP_ARGS=-vendor-only VERSION=${version}

RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make build-go GO_OUTDIR= VERSION=${version}

##########

FROM confluent-docker.jfrog.io/confluentinc/cc-base:v5.1.0-jdk-14

ARG version

ARG confluent_version

ARG git_sha

ARG git_branch

ENV COMPONENT=kafka

ENV KAFKA_SECRETS_DIR=/mnt/secrets

ENV KAFKA_LOG4J_DIR=/mnt/log

ENV KAFKA_CONFIG_DIR=/mnt/config

EXPOSE 9092

VOLUME ["${KAFKA_SECRETS_DIR}", "${KAFKA_LOG4J_DIR}"]

LABEL git_sha="${git_sha}"

LABEL git_branch="${git_branch}"

CMD ["/opt/caas/bin/run"]

#Copy kafka

COPY --from=kafka-builder /build /opt/confluent

COPY include/opt/caas /opt/caas

# Set up storage probe

COPY --from=go-build /storage-probe /opt/caas/bin

WORKDIR /

RUN mkdir -p /opt/caas/lib \

  && curl -o /opt/caas/lib/jmx_prometheus_javaagent-0.12.0.jar -O https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar \

  && mkdir -p /opt/asyncprofiler \

  && curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v1.6/async-profiler-1.6-linux-x64.tar.gz | tar xz -C /opt/asyncprofiler \

  && apt update \

  && apt install -y cc-rollingupgrade-ctl=0.9.0 vim-tiny \

  && apt-get autoremove -y \

  && mkdir -p  "${KAFKA_SECRETS_DIR}" "${KAFKA_LOG4J_DIR}" /opt/caas/config/kafka \

  && ln -s "${KAFKA_CONFIG_DIR}/kafka.properties" /opt/caas/config/kafka/kafka.properties \

  && chmod -R ag+w "${KAFKA_SECRETS_DIR}" "${KAFKA_LOG4J_DIR}"

#!/usr/bin/env groovy

/*

 * Licensed to the Apache Software Foundation (ASF) under one or more

 * contributor license agreements. See the NOTICE file distributed with

 * this work for additional information regarding copyright ownership.

 * The ASF licenses this file to You under the Apache License, Version 2.0

 * (the "License"); you may not use this file except in compliance with

 * the License. You may obtain a copy of the License at

 *

 *    http://www.apache.org/licenses/LICENSE-2.0

 *

 * Unless required by applicable law or agreed to in writing, software

 * distributed under the License is distributed on an "AS IS" BASIS,

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

 * See the License for the specific language governing permissions and

 * limitations under the License.

 */

def config = jobConfig {

    cron = '@weekly'

    nodeLabel = 'docker-oraclejdk8-ce-kafka'

    testResultSpecs = ['junit': '**/build/test-results/**/TEST-*.xml']

    slackChannel = '#kafka-warn'

    timeoutHours = 4

    runMergeCheck = false

    downStreamValidate = false

    downStreamRepos = ["common",]

    disableConcurrentBuilds = true

}

def retryFlagsString(jobConfig) {

    if (jobConfig.isPrJob) " -PmaxTestRetries=1 -PmaxTestRetryFailures=5"

    else ""

}

def downstreamBuildFailureOutput = ""

def publishStep(String mavenUrl) {

    sh "./gradlewAll -PmavenUrl=${mavenUrl} --no-daemon uploadArchives"

}

def job = {

    withVaultEnv([["artifactory/tools_jenkins", "user", "ORG_GRADLE_PROJECT_mavenUsername"],

        ["artifactory/tools_jenkins", "password", "ORG_GRADLE_PROJECT_mavenPassword"]]) {

        if (!config.isReleaseJob) {

            ciTool("ci-update-version ${env.WORKSPACE} ce-kafka")

        }

        stage("Check compilation compatibility with Scala 2.12") {

            sh "./gradlew clean assemble spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain " +

                    "--no-daemon --stacktrace -PxmlSpotBugsReport=true -PscalaVersion=2.12"

        }

        stage("Compile and validate") {

            sh "./gradlew clean assemble install spotlessScalaCheck checkstyleMain checkstyleTest spotbugsMain " +

                    "--no-daemon --stacktrace -PxmlSpotBugsReport=true"

        }

        if (config.publish && (config.isDevJob || config.isPreviewJob)) {

            stage("Publish to Artifactory") {

                if (!config.isReleaseJob && !config.isPrJob && !config.isPreviewJob) {

                    ciTool("ci-push-tag ${env.WORKSPACE} ce-kafka")

                }

                if (config.isDevJob) {

                    publishStep('https://confluent.jfrog.io/confluent/maven-public/')

                } else if (config.isPreviewJob) {

                    publishStep('https://confluent.jfrog.io/confluent/maven-releases-preview/')

                }

            }

        }

        if (config.publish && config.isDevJob && !config.isReleaseJob && !config.isPrJob) {

            stage("Start Downstream Builds") {

                config.downStreamRepos.each { repo ->

                    build(job: "confluentinc/${repo}/${env.BRANCH_NAME}",

                        wait: false,

                        propagate: false

                    )

                }

            }

        }

    }

    def runTestsStepName = "Step run-tests"

    def downstreamBuildsStepName = "Step cp-downstream-builds"

    def testTargets = [

        runTestsStepName: {

            stage('Run tests') {

                echo "Running unit and integration tests"

                sh "./gradlew unitTest integrationTest " +

                    "--no-daemon --stacktrace --continue -PtestLoggingEvents=started,passed,skipped,failed " +

                    "-PmaxParallelForks=4 -PignoreFailures=true" + retryFlagsString(config)

            }

            stage('Upload results') {

                // Kafka failed test stdout files

                archiveArtifacts artifacts: '**/testOutput/*.stdout', allowEmptyArchive: true

                def summary = junit '**/build/test-results/**/TEST-*.xml'

                def total = summary.getTotalCount()

                def failed = summary.getFailCount()

                def skipped = summary.getSkipCount()

                summary = "Test results:\n\t"

                summary = summary + ("Passed: " + (total - failed - skipped))

                summary = summary + (", Failed: " + failed)

                summary = summary + (", Skipped: " + skipped)

                return summary;

            }

        },

        downstreamBuildsStepName: {

            echo "Building cp-downstream-builds"

            stage('Downstream validation') {

                if (config.isPrJob && config.downStreamValidate) {

                    downStreamValidation(true, true)

                } else {

                    return "skip downStreamValidation"

                }

            }

        }

    ]

    result = parallel testTargets

    // combine results of the two targets into one result string

    return result.runTestsStepName + "\n" + result.downstreamBuildsStepName

}

runJob config, job

echo downstreamBuildFailureOutput

# Licensed to the Apache Software Foundation (ASF) under one or more

# contributor license agreements.  See the NOTICE file distributed with

# this work for additional information regarding copyright ownership.

# The ASF licenses this file to You under the Apache License, Version 2.0

# (the "License"); you may not use this file except in compliance with

# the License.  You may obtain a copy of the License at

#

#    http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

IMAGE_NAME := ce-kafka

BASE_IMAGE := confluent-docker.jfrog.io/confluentinc/cc-base

BASE_VERSION := v2.4.0

MASTER_BRANCH := master

KAFKA_VERSION := $(shell awk 'sub(/.*version=/,""){print $1}' ./gradle.properties)

VERSION_POST := -$(KAFKA_VERSION)

DOCKER_BUILD_PRE  += copy-gradle-properties

DOCKER_BUILD_POST += clean-gradle-properties

BUILD_TARGETS += build-docker-cc-kafka-init

BUILD_TARGETS += build-docker-cc-zookeeper

TEST_TARGETS += test-cc-services

RELEASE_POSTCOMMIT += push-docker-cc-kafka-init

RELEASE_POSTCOMMIT += push-docker-cc-zookeeper

ifeq ($(CONFLUENT_PLATFORM_PACKAGING),)

include ./mk-include/cc-begin.mk

include ./mk-include/cc-vault.mk

include ./mk-include/cc-semver.mk

include ./mk-include/cc-docker.mk

include ./mk-include/cc-end.mk

else

.PHONY: clean

clean:

.PHONY: distclean

distclean:

%:

	$(MAKE) -f debian/Makefile $@

endif

# Custom docker targets

.PHONY: show-docker-all

show-docker-all:

	@echo

	@echo ========================

	@echo "Docker info for ce-kafka:"

	@make VERSION=$(VERSION) show-docker

	@echo

	@echo ========================

	@echo "Docker info for cc-zookeeper"

	@make VERSION=$(VERSION) -C cc-zookeeper show-docker

	@echo

	@echo ========================

	@echo "Docker info for cc-kafka-init"

	@make VERSION=$(VERSION) -C cc-kafka-init show-docker

	@echo

	@echo ========================

	@echo "Docker info for soak_cluster"

	@make VERSION=$(VERSION) -C cc-services/soak_cluster show-docker

	@echo

	@echo ========================

	@echo "Docker info for trogdor"

	@make VERSION=$(VERSION) -C cc-services/trogdor show-docker

	@echo

	@echo ========================

	@echo "Docker info for tier-validator-services"

	@make VERSION=$(VERSION) -C cc-services/tier_validator show-docker

.PHONY: build-docker-cc-kafka-init

build-docker-cc-kafka-init:

	make VERSION=$(VERSION) -C cc-kafka-init build-docker

.PHONY: push-docker-cc-kafka-init

push-docker-cc-kafka-init:

	make VERSION=$(VERSION) -C cc-kafka-init push-docker

.PHONY: build-docker-cc-services

build-docker-cc-services:

	make VERSION=$(VERSION) BASE_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME) BASE_VERSION=$(IMAGE_VERSION) -C cc-services/soak_cluster build-docker

	make VERSION=$(VERSION) BASE_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME) BASE_VERSION=$(IMAGE_VERSION) -C cc-services/trogdor build-docker

	make VERSION=$(VERSION) BASE_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME) BASE_VERSION=$(IMAGE_VERSION) -C cc-services/tier_validator build-docker

.PHONY: push-docker-cc-services

push-docker-cc-services:

	make VERSION=$(VERSION) BASE_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME) BASE_VERSION=$(IMAGE_VERSION) -C cc-services/soak_cluster push-docker

	make VERSION=$(VERSION) BASE_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME) BASE_VERSION=$(IMAGE_VERSION) -C cc-services/trogdor push-docker

	make VERSION=$(VERSION) BASE_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME) BASE_VERSION=$(IMAGE_VERSION) -C cc-services/tier_validator push-docker

.PHONY: test-cc-services

test-cc-services:

	make VERSION=$(VERSION) -C cc-services/storage_probe test

.PHONY: build-docker-cc-zookeeper

build-docker-cc-zookeeper:

	make VERSION=$(VERSION) BASE_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME) BASE_VERSION=$(IMAGE_VERSION) -C cc-zookeeper build-docker

.PHONY: push-docker-cc-zookeeper

push-docker-cc-zookeeper:

	make VERSION=$(VERSION) BASE_IMAGE=$(IMAGE_REPO)/$(IMAGE_NAME) BASE_VERSION=$(IMAGE_VERSION) -C cc-zookeeper push-docker

GRADLE_TEMP = ./tmp/gradle/

.PHONY: copy-gradle-properties

copy-gradle-properties:

	mkdir -p $(GRADLE_TEMP)

	cp ~/.gradle/gradle.properties $(GRADLE_TEMP)

.PHONY: clean-gradle-properties

clean-gradle-properties:

	rm -rf $(GRADLE_TEMP)

This distribution has a binary dependency on jersey, which is available under the CDDL

License. The source code of jersey can be found at https://github.com/jersey/jersey/.

This distribution uses SSLExplorer (https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/samples/sni/SSLExplorer.java)

and SSLCapabilities (https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/samples/sni/SSLCapabilities.java),

with modification and refactored to clients/src/main/java/org/apache/kafka/common/network/SslUtil.java.

Both are available under the BSD 3-Clause License as described below:

/*

 * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.

 *

 * Redistribution and use in source and binary forms, with or without

 * modification, are permitted provided that the following conditions

 * are met:

 *

 *   - Redistributions of source code must retain the above copyright

 *     notice, this list of conditions and the following disclaimer.

 *

 *   - Redistributions in binary form must reproduce the above copyright

 *     notice, this list of conditions and the following disclaimer in the

 *     documentation and/or other materials provided with the distribution.

 *

 *   - Neither the name of Oracle or the names of its

 *     contributors may be used to endorse or promote products derived

 *     from this software without specific prior written permission.

 *

 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS

 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,

 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR

 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,

 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR

 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF

 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING

 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS

 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */

*More detailed description of your change,

if necessary. The PR title and PR message become

the squashed commit message, so use a separate

comment to ping reviewers.*

comment to ping reviewers. Please delete this

explanatory text.*

*Summary of testing strategy (including rationale)

for the feature or bug fix. Unit and/or integration

tests are expected for any behaviour change and

system tests should be considered for larger changes.*