这是本节的多页打印视图。点击此处打印.

dapr仓库的构建学习

Dapr构建学习之dapr仓库

1: 构建简介
2: Makefile文件
3: Makefile codegen部分
4: Makefile docker部分
5: 测试相关的构建学习

5.1: Makefile tests部分
5.2: 端到端测试部分
5.3: 性能测试部分

https://github.com/dapr/dapr

1 - 构建简介

dapr仓库是最核心的仓库

https://github.com/dapr/dapr

dapr 仓库是最核心的仓库，最重要的是二进制包是在 dapr 仓库中构建出来的。

2 - Makefile文件

定义和获取变量，定义各种 target

https://github.com/dapr/dapr/blob/master/Makefile

变量定义

和 kit/components-contrib 仓库一致内容就跳过了，只看新增的内容。

# 需要打包的二进制模块，默认是这五个
BINARIES    ?= daprd placement operator injector sentry
# 默认不开启 HA 模式
HA_MODE     ?= false
# Force in-memory log for placement
FORCE_INMEM ?= true

# Add latest tag if LATEST_RELEASE is true
LATEST_RELEASE ?=

PROTOC ?=protoc
# name of protoc-gen-go when protoc-gen-go --version is run.
PROTOC_GEN_GO_NAME = "protoc-gen-go"
ifdef REL_VERSION
	DAPR_VERSION := $(REL_VERSION)
else
	DAPR_VERSION := edge
endif

和二进制打包相关的定义：

ifeq ($(GOARCH),amd64)
	LATEST_TAG=latest
else
	LATEST_TAG=latest-$(GOARCH)
endif

LOCAL_OS := $(shell uname)
ifeq ($(LOCAL_OS),Linux)
   TARGET_OS_LOCAL = linux
else ifeq ($(LOCAL_OS),Darwin)
   TARGET_OS_LOCAL = darwin
else
   TARGET_OS_LOCAL ?= windows
   PROTOC_GEN_GO_NAME := "protoc-gen-go.exe"
endif
export GOOS ?= $(TARGET_OS_LOCAL)

PROTOC_GEN_GO_NAME+= "v1.26.0"

# Default docker container and e2e test targst.
TARGET_OS ?= linux
TARGET_ARCH ?= amd64
TEST_OUTPUT_FILE_PREFIX ?= ./test_report

ifeq ($(GOOS),windows)
BINARY_EXT_LOCAL:=.exe
GOLANGCI_LINT:=golangci-lint.exe
export ARCHIVE_EXT = .zip
else
BINARY_EXT_LOCAL:=
GOLANGCI_LINT:=golangci-lint
export ARCHIVE_EXT = .tar.gz
endif

export BINARY_EXT ?= $(BINARY_EXT_LOCAL)

OUT_DIR := ./dist

和 helm 相关的定义：

# Helm template and install setting
HELM:=helm
RELEASE_NAME?=dapr
DAPR_NAMESPACE?=dapr-system
DAPR_MTLS_ENABLED?=true
HELM_CHART_ROOT:=./charts
HELM_CHART_DIR:=$(HELM_CHART_ROOT)/dapr
HELM_OUT_DIR:=$(OUT_DIR)/install
HELM_MANIFEST_FILE:=$(HELM_OUT_DIR)/$(RELEASE_NAME).yaml
HELM_REGISTRY?=daprio.azurecr.io

和 go build 相关的定义：

################################################################################
# Go build details                                                             #
################################################################################
BASE_PACKAGE_NAME := github.com/dapr/dapr
LOGGER_PACKAGE_NAME := github.com/dapr/kit/logger

DEFAULT_LDFLAGS:=-X $(BASE_PACKAGE_NAME)/pkg/version.gitcommit=$(GIT_COMMIT) \
  -X $(BASE_PACKAGE_NAME)/pkg/version.gitversion=$(GIT_VERSION) \
  -X $(BASE_PACKAGE_NAME)/pkg/version.version=$(DAPR_VERSION) \
  -X $(LOGGER_PACKAGE_NAME).DaprVersion=$(DAPR_VERSION)
  
  ifeq ($(origin DEBUG), undefined)
  BUILDTYPE_DIR:=release
  LDFLAGS:="$(DEFAULT_LDFLAGS) -s -w"
else ifeq ($(DEBUG),0)
  BUILDTYPE_DIR:=release
  LDFLAGS:="$(DEFAULT_LDFLAGS) -s -w"
else
  BUILDTYPE_DIR:=debug
  GCFLAGS:=-gcflags="all=-N -l"
  LDFLAGS:="$(DEFAULT_LDFLAGS)"
  $(info Build with debugger information)
endif

DAPR_OUT_DIR := $(OUT_DIR)/$(GOOS)_$(GOARCH)/$(BUILDTYPE_DIR)
DAPR_LINUX_OUT_DIR := $(OUT_DIR)/linux_$(GOARCH)/$(BUILDTYPE_DIR)

build target

然后就是正式的 build target 定义了。

其中 target lint / target check-diff 和 components-contrib 仓库完全一致，跳过。

Target: build

# 前面的 BINARIES 定义
BINARIES    ?= daprd placement operator injector sentry
BUILDTYPE_DIR:=release 或着 debug
export BINARY_EXT ?= $(BINARY_EXT_LOCAL)  # 在windows 上构建时 BINARY_EXT_LOCAL 才会设置为 .exe
################################################################################
# Target: build                                                                #
################################################################################
.PHONY: build
DAPR_BINS:=$(foreach ITEM,$(BINARIES),$(DAPR_OUT_DIR)/$(ITEM)$(BINARY_EXT))
build: $(DAPR_BINS)

# Generate builds for dapr binaries for the target
# Params:
# $(1): the binary name for the target
# $(2): the binary main directory
# $(3): the target os
# $(4): the target arch
# $(5): the output directory
define genBinariesForTarget
.PHONY: $(5)/$(1)
$(5)/$(1):
	CGO_ENABLED=$(CGO) GOOS=$(3) GOARCH=$(4) go build $(GCFLAGS) -ldflags=$(LDFLAGS) \
	-o $(5)/$(1) $(2)/;
endef

# Generate binary targets
$(foreach ITEM,$(BINARIES),$(eval $(call genBinariesForTarget,$(ITEM)$(BINARY_EXT),./cmd/$(ITEM),$(GOOS),$(GOARCH),$(DAPR_OUT_DIR))))

循环构建 BINARIES 中定义的二进制模块。

在 macos + m1 芯片的 MacBook 上，输出如下：

make build 
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/darwin_arm64/release/daprd ./cmd/daprd/;
go: downloading github.com/dapr/components-contrib v1.6.0-rc.1.0.20220307041340-f1209fb068c7
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/darwin_arm64/release/placement ./cmd/placement/;
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/darwin_arm64/release/operator ./cmd/operator/;
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/darwin_arm64/release/injector ./cmd/injector/;
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/darwin_arm64/release/sentry ./cmd/sentry/;

构建成功的二进制文件存在放 ./dist/darwin_arm64/release/ 目录下:

Target: build-linux

################################################################################
# Target: build-linux                                                          #
################################################################################
BUILD_LINUX_BINS:=$(foreach ITEM,$(BINARIES),$(DAPR_LINUX_OUT_DIR)/$(ITEM))
build-linux: $(BUILD_LINUX_BINS)

# Generate linux binaries targets to build linux docker image
ifneq ($(GOOS), linux)
$(foreach ITEM,$(BINARIES),$(eval $(call genBinariesForTarget,$(ITEM),./cmd/$(ITEM),linux,$(GOARCH),$(DAPR_LINUX_OUT_DIR))))
endif

在 macos + m1 芯片的 MacBook 上，输出如下：

$ make build-linux 
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/linux_arm64/release/daprd ./cmd/daprd/;                                                                                
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/linux_arm64/release/placement ./cmd/placement/;
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/linux_arm64/release/operator ./cmd/operator/;
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/linux_arm64/release/injector ./cmd/injector/;
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/linux_arm64/release/sentry ./cmd/sentry/;

构建成功的二进制文件存在放 ./dist/linux_arm64/release/ 目录下:

注意是 arm64，不是 amd64.

Target: archive

# 相关的变量定义
OUT_DIR := ./dist
DAPR_OUT_DIR := $(OUT_DIR)/$(GOOS)_$(GOARCH)/$(BUILDTYPE_DIR)
DAPR_LINUX_OUT_DIR := $(OUT_DIR)/linux_$(GOARCH)/$(BUILDTYPE_DIR)

################################################################################
# Target: archive                                                              #
################################################################################
ARCHIVE_OUT_DIR ?= $(DAPR_OUT_DIR)
ARCHIVE_FILE_EXTS:=$(foreach ITEM,$(BINARIES),archive-$(ITEM)$(ARCHIVE_EXT))

archive: $(ARCHIVE_FILE_EXTS)

# Generate archive files for each binary
# $(1): the binary name to be archived
# $(2): the archived file output directory
define genArchiveBinary
ifeq ($(GOOS),windows)
archive-$(1).zip:
	7z.exe a -tzip "$(2)\\$(1)_$(GOOS)_$(GOARCH)$(ARCHIVE_EXT)" "$(DAPR_OUT_DIR)\\$(1)$(BINARY_EXT)"
else
archive-$(1).tar.gz:
	tar czf "$(2)/$(1)_$(GOOS)_$(GOARCH)$(ARCHIVE_EXT)" -C "$(DAPR_OUT_DIR)" "$(1)$(BINARY_EXT)"
endif
endef

# Generate archive-*.[zip|tar.gz] targets
$(foreach ITEM,$(BINARIES),$(eval $(call genArchiveBinary,$(ITEM),$(ARCHIVE_OUT_DIR))))

打包为 ta r.gz 或者 zip 文件。

在 macos + m1 芯片的 MacBook 上，输出如下：

$ make archive    
tar czf "./dist/darwin_arm64/release/daprd_darwin_arm64.tar.gz" -C "./dist/darwin_arm64/release" "daprd"
tar czf "./dist/darwin_arm64/release/placement_darwin_arm64.tar.gz" -C "./dist/darwin_arm64/release" "placement"
tar czf "./dist/darwin_arm64/release/operator_darwin_arm64.tar.gz" -C "./dist/darwin_arm64/release" "operator"
tar czf "./dist/darwin_arm64/release/injector_darwin_arm64.tar.gz" -C "./dist/darwin_arm64/release" "injector"
tar czf "./dist/darwin_arm64/release/sentry_darwin_arm64.tar.gz" -C "./dist/darwin_arm64/release" "sentry"

Target: manifest-gen

################################################################################
# Target: manifest-gen                                                         #
################################################################################

# Generate helm chart manifest
manifest-gen: dapr.yaml

dapr.yaml: check-docker-env
	$(info Generating helm manifest $(HELM_MANIFEST_FILE)...)
	@mkdir -p $(HELM_OUT_DIR)
	$(HELM) template \
		--include-crds=true  --set global.ha.enabled=$(HA_MODE) --set dapr_config.dapr_config_chart_included=false --set-string global.tag=$(DAPR_TAG) --set-string global.registry=$(DAPR_REGISTRY) $(HELM_CHART_DIR) > $(HELM_MANIFEST_FILE)

需要先设置 DAPR_REGISTRY 变量，如果没有设置则会报错：

make manifest-gen 
docker/docker.mk:76: *** DAPR_REGISTRY environment variable must be set.  Stop.

Target: Target: upload-helmchart

################################################################################
# Target: upload-helmchart
################################################################################

# Upload helm charts to Helm Registry
upload-helmchart:
	export HELM_EXPERIMENTAL_OCI=1; \
	$(HELM) chart save ${HELM_CHART_ROOT}/${RELEASE_NAME} ${HELM_REGISTRY}/${HELM}/${RELEASE_NAME}:${DAPR_VERSION}; \
	$(HELM) chart push ${HELM_REGISTRY}/${HELM}/${RELEASE_NAME}:${DAPR_VERSION}

TBD

Target: docker-deploy-k8s

################################################################################
# Target: docker-deploy-k8s                                                    #
################################################################################

PULL_POLICY?=Always
docker-deploy-k8s: check-docker-env check-arch
	$(info Deploying ${DAPR_REGISTRY}/${RELEASE_NAME}:${DAPR_TAG} to the current K8S context...)
	$(HELM) install \
		$(RELEASE_NAME) --namespace=$(DAPR_NAMESPACE) --wait --timeout 5m0s \
		--set global.ha.enabled=$(HA_MODE) --set-string global.tag=$(DAPR_TAG)-$(TARGET_OS)-$(TARGET_ARCH) \
		--set-string global.registry=$(DAPR_REGISTRY) --set global.logAsJson=true \
		--set global.daprControlPlaneOs=$(TARGET_OS) --set global.daprControlPlaneArch=$(TARGET_ARCH) \
		--set dapr_placement.logLevel=debug --set dapr_sidecar_injector.sidecarImagePullPolicy=$(PULL_POLICY) \
		--set global.imagePullPolicy=$(PULL_POLICY) --set global.imagePullSecrets=${DAPR_TEST_REGISTRY_SECRET} \
		--set global.mtls.enabled=${DAPR_MTLS_ENABLED} \
		--set dapr_placement.cluster.forceInMemoryLog=$(FORCE_INMEM) $(HELM_CHART_DIR)

TBD

Target: docker-deploy-k8s

################################################################################
# Target: docker-deploy-k8s                                                    #
################################################################################

PULL_POLICY?=Always
docker-deploy-k8s: check-docker-env check-arch
	$(info Deploying ${DAPR_REGISTRY}/${RELEASE_NAME}:${DAPR_TAG} to the current K8S context...)
	$(HELM) install \
		$(RELEASE_NAME) --namespace=$(DAPR_NAMESPACE) --wait --timeout 5m0s \
		--set global.ha.enabled=$(HA_MODE) --set-string global.tag=$(DAPR_TAG)-$(TARGET_OS)-$(TARGET_ARCH) \
		--set-string global.registry=$(DAPR_REGISTRY) --set global.logAsJson=true \
		--set global.daprControlPlaneOs=$(TARGET_OS) --set global.daprControlPlaneArch=$(TARGET_ARCH) \
		--set dapr_placement.logLevel=debug --set dapr_sidecar_injector.sidecarImagePullPolicy=$(PULL_POLICY) \
		--set global.imagePullPolicy=$(PULL_POLICY) --set global.imagePullSecrets=${DAPR_TEST_REGISTRY_SECRET} \
		--set global.mtls.enabled=${DAPR_MTLS_ENABLED} \
		--set dapr_placement.cluster.forceInMemoryLog=$(FORCE_INMEM) $(HELM_CHART_DIR)

TBD

Target: test

################################################################################
# Target: test                                                                 #
################################################################################
.PHONY: test
test: test-deps
	gotestsum --jsonfile $(TEST_OUTPUT_FILE_PREFIX)_unit.json --format standard-quiet -- ./pkg/... ./utils/... ./cmd/... $(COVERAGE_OPTS)
	go test ./tests/...

TBD

在 macos + m1 芯片的 MacBook 上，输出如下：

make test        
# The desire here is to download this test dependency without polluting go.mod
# In golang >=1.16 there is a new way to do this with `go install gotest.tools/gotestsum@latest`
# But this doesn't work with <=1.15.
# (see: https://golang.org/ref/mod#go-install)
command -v gotestsum || go install gotest.tools/gotestsum@latest
/Users/sky/work/soft/gopath/bin/gotestsum
gotestsum --jsonfile ./test_report_unit.json --format standard-quiet -- ./pkg/... ./utils/... ./cmd/... 
ok      github.com/dapr/dapr/pkg/acl    0.850s
ok      github.com/dapr/dapr/pkg/apis/subscriptions/v2alpha1    (cached)
ok      github.com/dapr/dapr/pkg/components/bindings    0.769s
ok      github.com/dapr/dapr/pkg/components/configuration       1.025s
......
ok      github.com/dapr/dapr/pkg/actors/internal        15.492s
ok      github.com/dapr/dapr/pkg/health 14.228s
ok      github.com/dapr/dapr/pkg/placement      10.674s
ok      github.com/dapr/dapr/pkg/sentry/ca      8.499s
......
# actor的测试实在是太夸张了
ok      github.com/dapr/dapr/pkg/actors 220.119s
......
DONE 1369 tests in 231.313s

Tareget: init-proto

################################################################################
# Target: init-proto                                                            #
################################################################################
.PHONY: init-proto
init-proto:
	go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26
	go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1

初始化安装 protoc-gen-go 和 protoc-gen-go-grpc 。

$ make init-proto
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26
go: downloading google.golang.org/protobuf v1.26.0
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1
go: downloading google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
go: downloading google.golang.org/protobuf v1.23.0

Tareget: gen-proto

################################################################################
# Target: gen-proto                                                            #
################################################################################
# 这是各个模块的 proto 文件 
GRPC_PROTOS:=common internals operator placement runtime sentry
PROTO_PREFIX:=github.com/dapr/dapr

# Generate archive files for each binary
# $(1): the binary name to be archived
define genProtoc
.PHONY: gen-proto-$(1)
gen-proto-$(1):
	$(PROTOC) --go_out=. --go_opt=module=$(PROTO_PREFIX) --go-grpc_out=. --go-grpc_opt=require_unimplemented_servers=false,module=$(PROTO_PREFIX) ./dapr/proto/$(1)/v1/*.proto
endef

$(foreach ITEM,$(GRPC_PROTOS),$(eval $(call genProtoc,$(ITEM))))

GEN_PROTOS:=$(foreach ITEM,$(GRPC_PROTOS),gen-proto-$(ITEM))

.PHONY: gen-proto
gen-proto: check-proto-version $(GEN_PROTOS) modtidy

Proto 源文件存放在 dapr/dapr 仓库下的 dapr/proto 目录中，子目录名对应 GRPC_PROTOS 变量。

注意在执行前要先安装好 protoc，否则会报错：

$ make gen-proto 
make: protoc: Command not found
please use protoc 3.14.0 to generate proto, see https://github.com/dapr/dapr/blob/master/dapr/README.md#proto-client-generation
make: *** [check-proto-version] Error 1

执行输出如下：

$ make gen-proto
protoc --go_out=. --go_opt=module=github.com/dapr/dapr --go-grpc_out=. --go-grpc_opt=require_unimplemented_servers=false,module=github.com/dapr/dapr ./dapr/proto/common/v1/*.proto
protoc --go_out=. --go_opt=module=github.com/dapr/dapr --go-grpc_out=. --go-grpc_opt=require_unimplemented_servers=false,module=github.com/dapr/dapr ./dapr/proto/internals/v1/*.proto
protoc --go_out=. --go_opt=module=github.com/dapr/dapr --go-grpc_out=. --go-grpc_opt=require_unimplemented_servers=false,module=github.com/dapr/dapr ./dapr/proto/operator/v1/*.proto
protoc --go_out=. --go_opt=module=github.com/dapr/dapr --go-grpc_out=. --go-grpc_opt=require_unimplemented_servers=false,module=github.com/dapr/dapr ./dapr/proto/placement/v1/*.proto
protoc --go_out=. --go_opt=module=github.com/dapr/dapr --go-grpc_out=. --go-grpc_opt=require_unimplemented_servers=false,module=github.com/dapr/dapr ./dapr/proto/runtime/v1/*.proto
protoc --go_out=. --go_opt=module=github.com/dapr/dapr --go-grpc_out=. --go-grpc_opt=require_unimplemented_servers=false,module=github.com/dapr/dapr ./dapr/proto/sentry/v1/*.proto
go mod tidy

生成的 go 文件会存放在 pkg/proto 目录下，子目录结构同样对应 GRPC_PROTOS 定义：

Target: check-proto-version

################################################################################
# Target: check-proto-version                                                         #
################################################################################
.PHONY: check-proto-version
check-proto-version: ## Checking the version of proto related tools
	@test "$(shell protoc --version)" = "libprotoc 3.14.0" \
	|| { echo "please use protoc 3.14.0 to generate proto, see https://github.com/dapr/dapr/blob/master/dapr/README.md#proto-client-generation"; exit 1; }

	@test "$(shell protoc-gen-go-grpc --version)" = "protoc-gen-go-grpc 1.1.0" \
	|| { echo "please use protoc-gen-go-grpc 1.1.0 to generate proto, see https://github.com/dapr/dapr/blob/master/dapr/README.md#proto-client-generation"; exit 1; }

	@test "$(shell protoc-gen-go --version 2>&1)" = "$(PROTOC_GEN_GO_NAME)" \
	|| { echo "please use protoc-gen-go v1.26.0 to generate proto, see https://github.com/dapr/dapr/blob/master/dapr/README.md#proto-client-generation"; exit 1; }

检查 proto 代码生成相关的工具的版本，因为不同版本生成的代码会有差异，如果混合不同版本使用，则这些生成的代码的 git 记录会混乱无比。

备注：这个 target 是我加的。

Target: check-proto-diff

################################################################################
# Target: check-proto-diff                                                           #
################################################################################
.PHONY: check-proto-diff
check-proto-diff:
	git diff --exit-code ./pkg/proto/common/v1/common.pb.go # check no changes
	git diff --exit-code ./pkg/proto/internals/v1/status.pb.go # check no changes
	git diff --exit-code ./pkg/proto/operator/v1/operator.pb.go # check no changes
	git diff --exit-code ./pkg/proto/operator/v1/operator_grpc.pb.go # check no changes
	git diff --exit-code ./pkg/proto/runtime/v1/appcallback.pb.go # check no changes
	git diff --exit-code ./pkg/proto/runtime/v1/appcallback_grpc.pb.go # check no changes
	git diff --exit-code ./pkg/proto/runtime/v1/dapr.pb.go # check no changes
	git diff --exit-code ./pkg/proto/runtime/v1/dapr_grpc.pb.go # check no changes
	git diff --exit-code ./pkg/proto/sentry/v1/sentry.pb.go # check no changes

检查 proto 生成的代码是否有变化。

$ make check-proto-diff   
git diff --exit-code ./pkg/proto/common/v1/common.pb.go # check no changes
git diff --exit-code ./pkg/proto/internals/v1/status.pb.go # check no changes
git diff --exit-code ./pkg/proto/operator/v1/operator.pb.go # check no changes
git diff --exit-code ./pkg/proto/operator/v1/operator_grpc.pb.go # check no changes
git diff --exit-code ./pkg/proto/runtime/v1/appcallback.pb.go # check no changes
git diff --exit-code ./pkg/proto/runtime/v1/appcallback_grpc.pb.go # check no changes
git diff --exit-code ./pkg/proto/runtime/v1/dapr.pb.go # check no changes
git diff --exit-code ./pkg/proto/runtime/v1/dapr_grpc.pb.go # check no changes
git diff --exit-code ./pkg/proto/sentry/v1/sentry.pb.go # check no changes

Target: get-components-contrib

################################################################################
# Target: get-components-contrib                                               #
################################################################################
.PHONY: get-components-contrib
get-components-contrib:
	go get github.com/dapr/components-contrib@master

获取 components-contrib 仓库 master 分支的最新代码。

执行结果输出如下：

$ make get-components-contrib 
go get github.com/dapr/components-contrib@master
go: downloading github.com/dapr/components-contrib v1.6.0-rc.1.0.20220310012151-027204f2d3c5
go get: upgraded github.com/dapr/components-contrib v1.6.0-rc.1.0.20220307041340-f1209fb068c7 => v1.6.0-rc.1.0.20220310012151-027204f2d3c5

3 - Makefile codegen部分

和代码生成相关的make target

https://github.com/dapr/dapr/blob/master/Makefile

################################################################################
# Target: codegen                                                              #
################################################################################
include tools/codegen.mk

https://github.com/dapr/dapr/blob/master/tools/codegen.mk

变量定义

# 依然是这六个 proto 模块
PROTOS = operator placement sentry common runtime internals

ifeq (,$(shell go env GOBIN))
GOBIN=$(shell go env GOPATH)/bin
else
GOBIN=$(shell go env GOBIN)
endif

Target: code-generate

# Generate code
code-generate: controller-gen
	$(CONTROLLER_GEN) object:headerFile="./tools/boilerplate.go.txt" \
		crd:trivialVersions=true crd:crdVersions=v1 paths="./pkg/apis/..." output:crd:artifacts:config=config/crd/bases

执行结果输出如下：

$ make code-generate   
go: downloading sigs.k8s.io/controller-tools v0.5.0
go: downloading github.com/spf13/cobra v1.1.1
go: downloading github.com/fatih/color v1.9.0
go: downloading golang.org/x/tools v0.0.0-20200616133436-c1934b75d054
go: downloading sigs.k8s.io/yaml v1.2.0
go: downloading k8s.io/api v0.20.2
go: downloading k8s.io/apimachinery v0.20.2
go: downloading gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
go: downloading k8s.io/apiextensions-apiserver v0.20.2
go: downloading github.com/gobuffalo/flect v0.2.2
go: downloading gopkg.in/yaml.v2 v2.3.0
go: downloading github.com/gogo/protobuf v1.3.1
go: downloading k8s.io/utils v0.0.0-20201110183641-67b214c5f920
go: downloading k8s.io/klog/v2 v2.4.0
go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.0.2
go: downloading golang.org/x/sys v0.0.0-20201112073958-5cba982894dd
go: downloading github.com/go-logr/logr v0.2.0
go: downloading github.com/json-iterator/go v1.1.10
go: downloading golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
go: downloading github.com/modern-go/reflect2 v1.0.1
go: downloading golang.org/x/text v0.3.4
go get: installing executables with 'go get' in module mode is deprecated.
        Use 'go install pkg@version' instead.
        For more information, see https://golang.org/doc/go-get-install-deprecation
        or run 'go help get' or 'go help install'.
/Users/sky/work/soft/gopath/bin/controller-gen object:headerFile="./tools/boilerplate.go.txt" \
                crd:trivialVersions=true crd:crdVersions=v1 paths="./pkg/apis/..." output:crd:artifacts:config=config/crd/bases

再次运行，在 controller-gen 已经下载好的情况下日志就干净很多了：

$ make code-generate 
/Users/sky/work/soft/gopath/bin/controller-gen object:headerFile="./tools/boilerplate.go.txt" \
                crd:trivialVersions=true crd:crdVersions=v1 paths="./pkg/apis/..." output:crd:artifacts:config=config/crd/bases

生成的代码存放在 dapr 项目根目录下的 config 子目录：

TBD：有空再细看。

Target: controller-gen

controller-gen target的定义有三部分。首先是找到 controller-gen，如果没有就下载：

# find or download controller-gen
# download controller-gen if necessary
controller-gen:
ifeq (, $(shell which controller-gen))
	@{ \
		set -e ;\
		CONTROLLER_GEN_TMP_DIR="$$(mktemp -d)" ;\
		cd "$$CONTROLLER_GEN_TMP_DIR" ;\
		GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.5.0 ; \
		rm -rf "$$CONTROLLER_GEN_TMP_DIR" ;\
	}
CONTROLLER_GEN=$(GOBIN)/controller-gen
else
CONTROLLER_GEN=$(shell which controller-gen)
endif

然后是一段方法定义, 动态定义了多个 protoc-gen-$(1)-v1 target：

define genProtoForTarget
.PHONY: $(1)
protoc-gen-$(1)-v1:
	protoc -I . ./dapr/proto/$(1)/v1/*.proto --go_out=plugins=grpc:.
	cp -R ./github.com/dapr/dapr/pkg/proto/$(1)/v1/*.go ./pkg/proto/$(1)/v1
	rm -rf ./github.com
endef

循环 PROTOS，为每个模块执行 protoc-gen-$(1)-v1 target：

PROTOS = operator placement sentry common runtime internals
# Generate proto gen targets
$(foreach ITEM,$(PROTOS),$(eval $(call genProtoForTarget,$(ITEM))))

PROTOC_ALL_TARGETS:=$(foreach ITEM,$(PROTOS),protoc-gen-$(ITEM)-v1)

protoc-gen: $(PROTOC_ALL_TARGETS)

4 - Makefile docker部分

和 docker 相关的 make target

https://github.com/dapr/dapr/blob/master/Makefile

################################################################################
# Target: docker                                                               #
################################################################################
include docker/docker.mk

https://github.com/dapr/dapr/blob/master/docker/docker.mk

变量定义

image相关的变量定义：

# Docker image build and push setting
DOCKER:=docker
DOCKERFILE_DIR?=./docker

DAPR_SYSTEM_IMAGE_NAME=$(RELEASE_NAME)
DAPR_RUNTIME_IMAGE_NAME=daprd
DAPR_PLACEMENT_IMAGE_NAME=placement
DAPR_SENTRY_IMAGE_NAME=sentry

# build docker image for linux
BIN_PATH=$(OUT_DIR)/$(TARGET_OS)_$(TARGET_ARCH)

ifeq ($(TARGET_OS), windows)
  DOCKERFILE:=Dockerfile-windows
  BIN_PATH := $(BIN_PATH)/release
else ifeq ($(origin DEBUG), undefined)
  DOCKERFILE:=Dockerfile
  BIN_PATH := $(BIN_PATH)/release
else ifeq ($(DEBUG),0)
  DOCKERFILE:=Dockerfile
  BIN_PATH := $(BIN_PATH)/release
else
  DOCKERFILE:=Dockerfile-debug
  BIN_PATH := $(BIN_PATH)/debug
endif

ifeq ($(TARGET_ARCH),arm)
  DOCKER_IMAGE_PLATFORM:=$(TARGET_OS)/arm/v7
else ifeq ($(TARGET_ARCH),arm64)
  DOCKER_IMAGE_PLATFORM:=$(TARGET_OS)/arm64/v8
else
  DOCKER_IMAGE_PLATFORM:=$(TARGET_OS)/amd64
endif

支持的 cpu 架构：

# Supported docker image architecture
DOCKERMUTI_ARCH=linux-amd64 linux-arm linux-arm64 windows-amd64

还有为 target docker-build 和 docker-push 定义的变量：

LINUX_BINS_OUT_DIR=$(OUT_DIR)/linux_$(GOARCH)
DOCKER_IMAGE_TAG=$(DAPR_REGISTRY)/$(DAPR_SYSTEM_IMAGE_NAME):$(DAPR_TAG)
DAPR_RUNTIME_DOCKER_IMAGE_TAG=$(DAPR_REGISTRY)/$(DAPR_RUNTIME_IMAGE_NAME):$(DAPR_TAG)
DAPR_PLACEMENT_DOCKER_IMAGE_TAG=$(DAPR_REGISTRY)/$(DAPR_PLACEMENT_IMAGE_NAME):$(DAPR_TAG)
DAPR_SENTRY_DOCKER_IMAGE_TAG=$(DAPR_REGISTRY)/$(DAPR_SENTRY_IMAGE_NAME):$(DAPR_TAG)

ifeq ($(LATEST_RELEASE),true)
DOCKER_IMAGE_LATEST_TAG=$(DAPR_REGISTRY)/$(DAPR_SYSTEM_IMAGE_NAME):$(LATEST_TAG)
DAPR_RUNTIME_DOCKER_IMAGE_LATEST_TAG=$(DAPR_REGISTRY)/$(DAPR_RUNTIME_IMAGE_NAME):$(LATEST_TAG)
DAPR_PLACEMENT_DOCKER_IMAGE_LATEST_TAG=$(DAPR_REGISTRY)/$(DAPR_PLACEMENT_IMAGE_NAME):$(LATEST_TAG)
DAPR_SENTRY_DOCKER_IMAGE_LATEST_TAG=$(DAPR_REGISTRY)/$(DAPR_SENTRY_IMAGE_NAME):$(LATEST_TAG)
endif


# To use buildx: https://github.com/docker/buildx#docker-ce
export DOCKER_CLI_EXPERIMENTAL=enabled

Target: check-docker-env

# check the required environment variables
check-docker-env:
ifeq ($(DAPR_REGISTRY),)
	$(error DAPR_REGISTRY environment variable must be set)
endif
ifeq ($(DAPR_TAG),)
	$(error DAPR_TAG environment variable must be set)
endif

检查 DAPR_REGISTRY 和 DAPR_TAG 两个环境变量是否设置，验证如下：

$ make check-docker-env
docker/docker.mk:76: *** DAPR_REGISTRY environment variable must be set.  Stop.
$ export DAPR_REGISTRY=docker.io/skyao
$ make check-docker-env               
docker/docker.mk:79: *** DAPR_TAG environment variable must be set.  Stop.
$ export DAPR_TAG=dev
$ make check-docker-env
make: Nothing to be done for `check-docker-env'.

Target: check-docker-env

check-arch:
ifeq ($(TARGET_OS),)
	$(error TARGET_OS environment variable must be set)
endif
ifeq ($(TARGET_ARCH),)
	$(error TARGET_ARCH environment variable must be set)
endif

检查 TARGET_OS 和 TARGET_ARCH 两个环境变量是否设置，验证如下：

$ make check-arch      
make: Nothing to be done for `check-arch'.

Target: docker-build

在执行 docker-build 之前，最好先设置好相关的环境变量：

export DAPR_REGISTRY=docker.io/skyao
export DAPR_TAG=dev			# 这个先别设置，用默认值
export TARGET_OS=linux		# 默认是linux
export TARGET_ARCH=arm64  # 默认是amd64，m1上需要修改

然后执行：

docker-build: check-docker-env check-arch
	$(info Building $(DOCKER_IMAGE_TAG) docker image ...)
ifeq ($(TARGET_ARCH),amd64)
	$(DOCKER) build --build-arg PKG_FILES=* -f $(DOCKERFILE_DIR)/$(DOCKERFILE) $(BIN_PATH) -t $(DOCKER_IMAGE_TAG)-$(TARGET_OS)-$(TARGET_ARCH)
	$(DOCKER) build --build-arg PKG_FILES=daprd -f $(DOCKERFILE_DIR)/$(DOCKERFILE) $(BIN_PATH) -t $(DAPR_RUNTIME_DOCKER_IMAGE_TAG)-$(TARGET_OS)-$(TARGET_ARCH)
	$(DOCKER) build --build-arg PKG_FILES=placement -f $(DOCKERFILE_DIR)/$(DOCKERFILE) $(BIN_PATH) -t $(DAPR_PLACEMENT_DOCKER_IMAGE_TAG)-$(TARGET_OS)-$(TARGET_ARCH)
	$(DOCKER) build --build-arg PKG_FILES=sentry -f $(DOCKERFILE_DIR)/$(DOCKERFILE) $(BIN_PATH) -t $(DAPR_SENTRY_DOCKER_IMAGE_TAG)-$(TARGET_OS)-$(TARGET_ARCH)
else
	-$(DOCKER) buildx create --use --name daprbuild
	-$(DOCKER) run --rm --privileged multiarch/qemu-user-static --reset -p yes
	$(DOCKER) buildx build --build-arg PKG_FILES=* --platform $(DOCKER_IMAGE_PLATFORM) -f $(DOCKERFILE_DIR)/$(DOCKERFILE) $(BIN_PATH) -t $(DOCKER_IMAGE_TAG)-$(TARGET_OS)-$(TARGET_ARCH)
	$(DOCKER) buildx build --build-arg PKG_FILES=daprd --platform $(DOCKER_IMAGE_PLATFORM) -f $(DOCKERFILE_DIR)/$(DOCKERFILE) $(BIN_PATH) -t $(DAPR_RUNTIME_DOCKER_IMAGE_TAG)-$(TARGET_OS)-$(TARGET_ARCH)
	$(DOCKER) buildx build --build-arg PKG_FILES=placement --platform $(DOCKER_IMAGE_PLATFORM) -f $(DOCKERFILE_DIR)/$(DOCKERFILE) $(BIN_PATH) -t $(DAPR_PLACEMENT_DOCKER_IMAGE_TAG)-$(TARGET_OS)-$(TARGET_ARCH)
	$(DOCKER) buildx build --build-arg PKG_FILES=sentry --platform $(DOCKER_IMAGE_PLATFORM) -f $(DOCKERFILE_DIR)/$(DOCKERFILE) $(BIN_PATH) -t $(DAPR_SENTRY_DOCKER_IMAGE_TAG)-$(TARGET_OS)-$(TARGET_ARCH)
endif

在执行 docker-build 之前，需要先执行 target build-linux，不然会提示：

$ make docker-build 
Building docker.io/skyao/dapr:dev docker image ...
docker build --build-arg PKG_FILES=* -f ./docker/Dockerfile ./dist/linux_amd64/release -t docker.io/skyao/dapr:dev-linux-amd64
unable to prepare context: path "./dist/linux_amd64/release" not found
make: *** [docker-build] Error 1

m1上的构建

target build-linux 会构建 linux_arm64 版本的二进制文件：

$ make build-linux 
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/linux_arm64/release/daprd ./cmd/daprd/;
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/linux_arm64/release/placement ./cmd/placement/;
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/linux_arm64/release/operator ./cmd/operator/;
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/linux_arm64/release/injector ./cmd/injector/;
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build  -ldflags="-X github.com/dapr/dapr/pkg/version.gitcommit=551722f533afa5dfee97482fe3e63d8ff6233d50 -X github.com/dapr/dapr/pkg/version.gitversion=v1.5.1-rc.3-411-g551722f -X github.com/dapr/dapr/pkg/version.version=edge -X github.com/dapr/kit/logger.DaprVersion=edge -s -w" -o ./dist/linux_arm64/release/sentry ./cmd/sentry/;

在 m1 上执行 docker-build ：

$ make docker-build     

Building docker.io/skyao/dapr:dev docker image ...
docker buildx create --use --name daprbuild
error: existing instance for daprbuild but no append mode, specify --node to make changes for existing instances
make: [docker-build] Error 1 (ignored)
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
Unable to find image 'multiarch/qemu-user-static:latest' locally
latest: Pulling from multiarch/qemu-user-static
01c2cdc13739: Pull complete 
11933eee4160: Pull complete 
30abb83a18eb: Pull complete 
0657daef200b: Pull complete 
10094524a9f3: Pull complete 
Digest: sha256:2c8b8fcf1d6badfca797c3fb46b7bb5f705ec7e66363e1cfeb7b7d4c7086e360
Status: Downloaded newer image for multiarch/qemu-user-static:latest
WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested
Error while loading /qemu-binfmt-conf.sh: Exec format error
make: [docker-build] Error 1 (ignored)
docker buildx build --build-arg PKG_FILES=* --platform linux/arm64/v8 -f ./docker/Dockerfile ./dist/linux_arm64/release -t docker.io/skyao/dapr:dev-linux-arm64
WARN[0000] No output specified for docker-container driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load 
[+] Building 84.8s (12/12) FINISHED                                                                                                                                                                             
 => [internal] booting buildkit                                                                                                                                                                           69.4s
 => => pulling image moby/buildkit:buildx-stable-1                                                                                                                                                        69.0s
 => => creating container buildx_buildkit_daprbuild0                                                                                                                                                       0.4s
 => [internal] load build definition from Dockerfile                                                                                                                                                       0.0s
 => => transferring dockerfile: 297B                                                                                                                                                                       0.0s
 => [internal] load .dockerignore                                                                                                                                                                          0.0s
 => => transferring context: 2B                                                                                                                                                                            0.0s
 => [internal] load metadata for gcr.io/distroless/static:nonroot                                                                                                                                          4.7s
 => [internal] load metadata for docker.io/library/alpine:latest                                                                                                                                           6.9s
 => [auth] library/alpine:pull token for registry-1.docker.io                                                                                                                                              0.0s
 => [alpine 1/2] FROM docker.io/library/alpine:latest@sha256:ceeae2849a425ef1a7e591d8288f1a58cdf1f4e8d9da7510e29ea829e61cf512                                                                              0.2s
 => => resolve docker.io/library/alpine:latest@sha256:ceeae2849a425ef1a7e591d8288f1a58cdf1f4e8d9da7510e29ea829e61cf512                                                                                     0.0s
 => => sha256:a5e44472bb1f0d721d23f82fa10a4c3d25994790238a173c1de950a649eb9a90 2.71MB / 2.71MB                                                                                                             2.9s
 => => extracting sha256:a5e44472bb1f0d721d23f82fa10a4c3d25994790238a173c1de950a649eb9a90                                                                                                                  0.2s
 => [internal] load build context                                                                                                                                                                          7.4s
 => => transferring context: 233.03MB                                                                                                                                                                      7.3s
 => [stage-1 1/4] FROM gcr.io/distroless/static:nonroot@sha256:80c956fb0836a17a565c43a4026c9c80b2013c83bea09f74fa4da195a59b7a99                                                                            0.2s
 => => resolve gcr.io/distroless/static:nonroot@sha256:80c956fb0836a17a565c43a4026c9c80b2013c83bea09f74fa4da195a59b7a99                                                                                    0.0s
 => => sha256:dbcab61d5a5a806aee6156f2e22c601a52119ca8eaeb8fcd08187f22c35d9b88 803.83kB / 803.83kB                                                                                                         3.6s
 => => extracting sha256:dbcab61d5a5a806aee6156f2e22c601a52119ca8eaeb8fcd08187f22c35d9b88                                                                                                                  0.2s
 => [alpine 2/2] RUN apk add -U --no-cache ca-certificates                                                                                                                                                 4.5s
 => [stage-1 2/4] COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/                                                                                                                    0.0s
 => [stage-1 3/4] COPY /* /                                                                                                                                                                                0.2s
docker buildx build --build-arg PKG_FILES=daprd --platform linux/arm64/v8 -f ./docker/Dockerfile ./dist/linux_arm64/release -t docker.io/skyao/daprd:dev-linux-arm64                                            
WARN[0000] No output specified for docker-container driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load             
[+] Building 1.3s (10/10) FINISHED                                                                                                                                                                              
 => [internal] load build definition from Dockerfile                                                                                                                                                       0.0s
 => => transferring dockerfile: 297B                                                                                                                                                                       0.0s
 => [internal] load .dockerignore                                                                                                                                                                          0.0s
 => => transferring context: 2B                                                                                                                                                                            0.0s
 => [internal] load metadata for gcr.io/distroless/static:nonroot                                                                                                                                          0.9s
 => [internal] load metadata for docker.io/library/alpine:latest                                                                                                                                           0.5s
 => [alpine 1/2] FROM docker.io/library/alpine:latest@sha256:ceeae2849a425ef1a7e591d8288f1a58cdf1f4e8d9da7510e29ea829e61cf512                                                                              0.0s
 => => resolve docker.io/library/alpine:latest@sha256:ceeae2849a425ef1a7e591d8288f1a58cdf1f4e8d9da7510e29ea829e61cf512                                                                                     0.0s
 => [internal] load build context                                                                                                                                                                          0.1s
 => => transferring context: 29B                                                                                                                                                                           0.0s
 => [stage-1 1/4] FROM gcr.io/distroless/static:nonroot@sha256:80c956fb0836a17a565c43a4026c9c80b2013c83bea09f74fa4da195a59b7a99                                                                            0.0s
 => => resolve gcr.io/distroless/static:nonroot@sha256:80c956fb0836a17a565c43a4026c9c80b2013c83bea09f74fa4da195a59b7a99                                                                                    0.0s
 => CACHED [alpine 2/2] RUN apk add -U --no-cache ca-certificates                                                                                                                                          0.0s
 => CACHED [stage-1 2/4] COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/                                                                                                             0.0s
 => [stage-1 3/4] COPY /daprd /                                                                                                                                                                            0.1s
docker buildx build --build-arg PKG_FILES=placement --platform linux/arm64/v8 -f ./docker/Dockerfile ./dist/linux_arm64/release -t docker.io/skyao/placement:dev-linux-arm64
WARN[0000] No output specified for docker-container driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load 
[+] Building 1.9s (10/10) FINISHED                                                                                                                                                                              
 => [internal] load build definition from Dockerfile                                                                                                                                                       0.0s
 => => transferring dockerfile: 297B                                                                                                                                                                       0.0s
 => [internal] load .dockerignore                                                                                                                                                                          0.0s
 => => transferring context: 2B                                                                                                                                                                            0.0s
 => [internal] load metadata for gcr.io/distroless/static:nonroot                                                                                                                                          1.2s
 => [internal] load metadata for docker.io/library/alpine:latest                                                                                                                                           0.5s
 => [alpine 1/2] FROM docker.io/library/alpine:latest@sha256:ceeae2849a425ef1a7e591d8288f1a58cdf1f4e8d9da7510e29ea829e61cf512                                                                              0.0s
 => => resolve docker.io/library/alpine:latest@sha256:ceeae2849a425ef1a7e591d8288f1a58cdf1f4e8d9da7510e29ea829e61cf512                                                                                     0.0s
 => [internal] load build context                                                                                                                                                                          0.5s
 => => transferring context: 14.42MB                                                                                                                                                                       0.5s
 => [stage-1 1/4] FROM gcr.io/distroless/static:nonroot@sha256:80c956fb0836a17a565c43a4026c9c80b2013c83bea09f74fa4da195a59b7a99                                                                            0.0s
 => => resolve gcr.io/distroless/static:nonroot@sha256:80c956fb0836a17a565c43a4026c9c80b2013c83bea09f74fa4da195a59b7a99                                                                                    0.0s
 => CACHED [alpine 2/2] RUN apk add -U --no-cache ca-certificates                                                                                                                                          0.0s
 => CACHED [stage-1 2/4] COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/                                                                                                             0.0s
 => [stage-1 3/4] COPY /placement /                                                                                                                                                                        0.0s
docker buildx build --build-arg PKG_FILES=sentry --platform linux/arm64/v8 -f ./docker/Dockerfile ./dist/linux_arm64/release -t docker.io/skyao/sentry:dev-linux-arm64
WARN[0000] No output specified for docker-container driver. Build result will only remain in the build cache. To push result image into registry use --push or to load image into docker use --load 
[+] Building 2.5s (10/10) FINISHED                                                                                                                                                                              
 => [internal] load build definition from Dockerfile                                                                                                                                                       0.0s
 => => transferring dockerfile: 297B                                                                                                                                                                       0.0s
 => [internal] load .dockerignore                                                                                                                                                                          0.0s
 => => transferring context: 2B                                                                                                                                                                            0.0s
 => [internal] load metadata for gcr.io/distroless/static:nonroot                                                                                                                                          1.0s
 => [internal] load metadata for docker.io/library/alpine:latest                                                                                                                                           0.5s
 => [alpine 1/2] FROM docker.io/library/alpine:latest@sha256:ceeae2849a425ef1a7e591d8288f1a58cdf1f4e8d9da7510e29ea829e61cf512                                                                              0.0s
 => => resolve docker.io/library/alpine:latest@sha256:ceeae2849a425ef1a7e591d8288f1a58cdf1f4e8d9da7510e29ea829e61cf512                                                                                     0.0s
 => [internal] load build context                                                                                                                                                                          1.3s
 => => transferring context: 36.64MB                                                                                                                                                                       1.3s
 => [stage-1 1/4] FROM gcr.io/distroless/static:nonroot@sha256:80c956fb0836a17a565c43a4026c9c80b2013c83bea09f74fa4da195a59b7a99                                                                            0.0s
 => => resolve gcr.io/distroless/static:nonroot@sha256:80c956fb0836a17a565c43a4026c9c80b2013c83bea09f74fa4da195a59b7a99                                                                                    0.0s
 => CACHED [alpine 2/2] RUN apk add -U --no-cache ca-certificates                                                                                                                                          0.0s
 => CACHED [stage-1 2/4] COPY --from=alpine /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/                                                                                                             0.0s
 => [stage-1 3/4] COPY /sentry /

amd64 上的构建

TBD

5 - 测试相关的构建学习

测试相关的构建学习

5.1 - Makefile tests部分

和测试相关的make target

https://github.com/dapr/dapr/blob/master/Makefile

################################################################################
# Target: tests                                                                #
################################################################################
include tests/dapr_tests.mk

https://github.com/dapr/dapr/blob/master/tests/dapr_tests.mk

变量定义

容器和k8s相关：

KUBECTL=kubectl

DAPR_CONTAINER_LOG_PATH?=./dist/container_logs

DAPR_TEST_SECONDARY_NAMESPACE=dapr-tests-2

# 测试用的 namespace，默认是 dapr-system，这个不合适，一般还是设置为 dapr-test 吧
ifeq ($(DAPR_TEST_NAMESPACE),)
DAPR_TEST_NAMESPACE=$(DAPR_NAMESPACE)
endif

ifeq ($(DAPR_TEST_REGISTRY),)
DAPR_TEST_REGISTRY=$(DAPR_REGISTRY)
endif

# 测试使用的镜像文件的 tag
ifeq ($(DAPR_TEST_TAG),)
DAPR_TEST_TAG=$(DAPR_TAG)-$(TARGET_OS)-$(TARGET_ARCH)
endif

# mimikube相关
ifeq ($(DAPR_TEST_ENV),minikube)
MINIKUBE_NODE_IP=$(shell minikube ip)
ifeq ($(MINIKUBE_NODE_IP),)
$(error cannot find get minikube node ip address. ensure that you have minikube environment.)
endif
endif

测试使用的 statue store 和 pub/sub 的默认值：

ifeq ($(DAPR_TEST_STATE_STORE),)
DAPR_TEST_STATE_STORE=redis
endif

ifeq ($(DAPR_TEST_QUERY_STATE_STORE),)
DAPR_TEST_QUERY_STATE_STORE=mongodb
endif

ifeq ($(DAPR_TEST_PUBSUB),)
DAPR_TEST_PUBSUB=redis
endif

检测当前os：

ifeq ($(OS),Windows_NT) 
    detected_OS := windows
else
    detected_OS := $(shell sh -c 'uname 2>/dev/null || echo Unknown' |  tr '[:upper:]' '[:lower:]')
endif

Target: get-components-contrib

################################################################################
# Target: get-components-contrib                                               #
################################################################################
.PHONY: get-components-contrib
get-components-contrib:
	go get github.com/dapr/components-contrib@master

获取 components-contrib 仓库 master 分支的最新代码。

执行结果输出如下：

$ make get-components-contrib 
go get github.com/dapr/components-contrib@master
go: downloading github.com/dapr/components-contrib v1.6.0-rc.1.0.20220310012151-027204f2d3c5
go get: upgraded github.com/dapr/components-contrib v1.6.0-rc.1.0.20220307041340-f1209fb068c7 => v1.6.0-rc.1.0.20220310012151-027204f2d3c5

Target: create-test-namespace

create-test-namespace:
	kubectl create namespace $(DAPR_TEST_NAMESPACE)
	kubectl create namespace $(DAPR_TEST_SECONDARY_NAMESPACE)

Target: delete-test-namespace

delete-test-namespace:
	kubectl delete namespace $(DAPR_TEST_NAMESPACE)
	kubectl delete namespace $(DAPR_TEST_SECONDARY_NAMESPACE)

Target: setup-3rd-party

setup-3rd-party: setup-helm-init setup-test-env-redis setup-test-env-kafka setup-test-env-mongodb

Target: test-deps

.PHONY: test-deps
test-deps:
	# The desire here is to download this test dependency without polluting go.mod
	# In golang >=1.16 there is a new way to do this with `go install gotest.tools/gotestsum@latest`
	# But this doesn't work with <=1.15.
	# (see: https://golang.org/ref/mod#go-install)
	command -v gotestsum || go install gotest.tools/gotestsum@latest

Target: setup-helm-init

# add required helm repo
setup-helm-init:
	$(HELM) repo add bitnami https://charts.bitnami.com/bitnami
	$(HELM) repo add stable https://charts.helm.sh/stable
	$(HELM) repo add incubator https://charts.helm.sh/incubator
	$(HELM) repo update

redis 相关的target

# install redis to the cluster without password
setup-test-env-redis:
	$(HELM) install dapr-redis bitnami/redis --wait --timeout 5m0s --namespace $(DAPR_TEST_NAMESPACE) -f ./tests/config/redis_override.yaml

delete-test-env-redis:
	${HELM} del dapr-redis --namespace ${DAPR_TEST_NAMESPACE}

kafka 相关的target

# install kafka to the cluster
setup-test-env-kafka:
	$(HELM) install dapr-kafka bitnami/kafka -f ./tests/config/kafka_override.yaml --namespace $(DAPR_TEST_NAMESPACE) --timeout 10m0s

# delete kafka from cluster
delete-test-env-kafka:
	$(HELM) del dapr-kafka --namespace $(DAPR_TEST_NAMESPACE)

mongodb 相关的target

# install mongodb to the cluster without password
setup-test-env-mongodb:
	$(HELM) install dapr-mongodb bitnami/mongodb -f ./tests/config/mongodb_override.yaml --namespace $(DAPR_TEST_NAMESPACE) --wait --timeout 5m0s

# delete mongodb from cluster
delete-test-env-mongodb:
	${HELM} del dapr-mongodb --namespace ${DAPR_TEST_NAMESPACE}

Target: setup-test-env

# Install redis and kafka to test cluster
setup-test-env: setup-test-env-kafka setup-test-env-redis setup-test-env-mongodb

保存k8s 资源和日志

save-dapr-control-plane-k8s-resources:
	mkdir -p '$(DAPR_CONTAINER_LOG_PATH)'
	kubectl describe all -n $(DAPR_TEST_NAMESPACE) > '$(DAPR_CONTAINER_LOG_PATH)/control_plane_k8s_resources.txt'

save-dapr-control-plane-k8s-logs:
	mkdir -p '$(DAPR_CONTAINER_LOG_PATH)'
	kubectl logs -l 'app.kubernetes.io/name=dapr' -n $(DAPR_TEST_NAMESPACE) > '$(DAPR_CONTAINER_LOG_PATH)/control_plane_containers.log'

Target: setup-disable-mtls

# Apply default config yaml to turn mTLS off for testing (mTLS is enabled by default)
setup-disable-mtls:
	$(KUBECTL) apply -f ./tests/config/dapr_mtls_off_config.yaml --namespace $(DAPR_TEST_NAMESPACE)

Target: setup-app-configurations

# Apply default config yaml to turn tracing off for testing (tracing is enabled by default)
setup-app-configurations:
	$(KUBECTL) apply -f ./tests/config/dapr_observability_test_config.yaml --namespace $(DAPR_TEST_NAMESPACE)

Target: setup-test-components

# Apply component yaml for state, secrets, pubsub, and bindings
setup-test-components: setup-app-configurations
	$(KUBECTL) apply -f ./tests/config/kubernetes_secret.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/kubernetes_secret_config.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/kubernetes_redis_secret.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/dapr_$(DAPR_TEST_STATE_STORE)_state.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/dapr_$(DAPR_TEST_QUERY_STATE_STORE)_state.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/dapr_tests_cluster_role_binding.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/dapr_$(DAPR_TEST_PUBSUB)_pubsub.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/dapr_kafka_bindings.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/dapr_kafka_bindings_custom_route.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/dapr_kafka_bindings_grpc.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/app_topic_subscription_pubsub.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/app_topic_subscription_pubsub_grpc.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/kubernetes_allowlists_config.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/kubernetes_allowlists_grpc_config.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/dapr_redis_state_query.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/dapr_redis_state_badhost.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/dapr_redis_state_badpass.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/uppercase.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/pipeline.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/app_reentrant_actor.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/app_actor_type_metadata.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/app_topic_subscription_routing.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/app_topic_subscription_routing_grpc.yaml --namespace $(DAPR_TEST_NAMESPACE)
	$(KUBECTL) apply -f ./tests/config/app_pubsub_routing.yaml --namespace $(DAPR_TEST_NAMESPACE)

	# Show the installed components
	$(KUBECTL) get components --namespace $(DAPR_TEST_NAMESPACE)

	# Show the installed configurations
	$(KUBECTL) get configurations --namespace $(DAPR_TEST_NAMESPACE)

Target: clean-test-env

# Clean up test environment
clean-test-env:
	./tests/test-infra/clean_up.sh $(DAPR_TEST_NAMESPACE)
	./tests/test-infra/clean_up.sh $(DAPR_TEST_NAMESPACE)-2

TODO: 这里应该用 DAPR_TEST_SECONDARY_NAMESPACE ，随手 PR： fix build target clean-test-env by skyao · Pull Request #4420 · dapr/dapr (github.com)

kind 相关的 target


# Setup kind
setup-kind:
	kind create cluster --config ./tests/config/kind.yaml --name kind
	kubectl cluster-info --context kind-kind
	# Setup registry
	docker run -d --restart=always -p 5000:5000 --name kind-registry registry:2
	# Connect the registry to the KinD network.
	docker network connect "kind" kind-registry
	# Setup metrics-server
	helm install ms stable/metrics-server -n kube-system --set=args={--kubelet-insecure-tls}

describe-kind-env:
	@echo "\
	export MINIKUBE_NODE_IP=`kubectl get nodes \
	    -lkubernetes.io/hostname!=kind-control-plane \
        -ojsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'`\n\
	export DAPR_REGISTRY=$${DAPR_REGISTRY:-localhost:5000/dapr}\n\
	export DAPR_TEST_REGISTRY=$${DAPR_TEST_REGISTRY:-localhost:5000/dapr}\n\
	export DAPR_TAG=dev\n\
	export DAPR_NAMESPACE=dapr-tests"
	

delete-kind:
	docker stop kind-registry && docker rm kind-registry || echo "Could not delete registry."
	kind delete cluster --name kind

minikube 相关的 target

setup-minikube-darwin:
	minikube start --memory=4g --cpus=4 --driver=hyperkit --kubernetes-version=v1.18.8
	minikube addons enable metrics-server

setup-minikube-windows:
	minikube start --memory=4g --cpus=4 --kubernetes-version=v1.18.8
	minikube addons enable metrics-server

setup-minikube-linux:
	minikube start --memory=4g --cpus=4 --kubernetes-version=v1.18.8
	minikube addons enable metrics-server

setup-minikube: setup-minikube-$(detected_OS)

describe-minikube-env:
	@echo "\
	export DAPR_REGISTRY=docker.io/`docker-credential-desktop list | jq -r '\
	. | to_entries[] | select(.key | contains("docker.io")) | last(.value)'`\n\
	export DAPR_TAG=dev\n\
	export DAPR_NAMESPACE=dapr-tests\n\
	export DAPR_TEST_ENV=minikube\n\
	export DAPR_TEST_REGISTRY=\n\
	export MINIKUBE_NODE_IP="

# Setup minikube
delete-minikube:
	minikube delete

5.2 - 端到端测试部分

和端到端测试相关的make target

E2e测试的 target 定义在 dapr_tests.mk：

https://github.com/dapr/dapr/blob/master/tests/dapr_tests.mk

但因为内容比较独立，单独拿出来细看

变量定义

E2E_TEST_APPS 定义了要 e2e 测试要用到的app：

# E2E test app list
# e.g. E2E_TEST_APPS=hellodapr state service_invocation
E2E_TEST_APPS=actorjava \
actordotnet \
actorpython \
actorphp \
hellodapr \
stateapp \
secretapp \
service_invocation \
service_invocation_grpc \
service_invocation_grpc_proxy_client \
service_invocation_grpc_proxy_server \
binding_input \
binding_input_grpc \
binding_output \
pubsub-publisher \
pubsub-subscriber \
pubsub-subscriber_grpc \
pubsub-subscriber-routing \
pubsub-subscriber-routing_grpc \
actorapp \
actorclientapp \
actorfeatures \
actorinvocationapp \
actorreentrancy \
runtime \
runtime_init \
middleware \
job-publisher \

E2E_TESTAPP_DIR 是 e2e 测试应用所在的目录：

# E2E test app root directory
E2E_TESTAPP_DIR=./tests/apps

Target: check-e2e-env

# check the required environment variables
check-e2e-env:
ifeq ($(DAPR_TEST_REGISTRY),)
	$(error DAPR_TEST_REGISTRY environment variable must be set)
endif
ifeq ($(DAPR_TEST_TAG),)
	$(error DAPR_TEST_TAG environment variable must be set)
endif

Target: build-e2e-app-x

define genTestAppImageBuild
.PHONY: build-e2e-app-$(1)
build-e2e-app-$(1): check-e2e-env
ifeq (,$(wildcard $(E2E_TESTAPP_DIR)/$(1)/$(DOCKERFILE)))
	CGO_ENABLED=0 GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -o $(E2E_TESTAPP_DIR)/$(1)/app$(BINARY_EXT_LOCAL) $(E2E_TESTAPP_DIR)/$(1)/app.go
	$(DOCKER) build -f $(E2E_TESTAPP_DIR)/$(DOCKERFILE) $(E2E_TESTAPP_DIR)/$(1)/. -t $(DAPR_TEST_REGISTRY)/e2e-$(1):$(DAPR_TEST_TAG)
else
	$(DOCKER) build -f $(E2E_TESTAPP_DIR)/$(1)/$(DOCKERFILE) $(E2E_TESTAPP_DIR)/$(1)/. -t $(DAPR_TEST_REGISTRY)/e2e-$(1):$(DAPR_TEST_TAG)
endif
endef

# Generate test app image build targets
$(foreach ITEM,$(E2E_TEST_APPS),$(eval $(call genTestAppImageBuild,$(ITEM))))

Target: push-e2e-app-x

define genTestAppImagePush
.PHONY: push-e2e-app-$(1)
push-e2e-app-$(1): check-e2e-env
	$(DOCKER) push $(DAPR_TEST_REGISTRY)/e2e-$(1):$(DAPR_TEST_TAG)
endef

# Generate test app image push targets
$(foreach ITEM,$(E2E_TEST_APPS),$(eval $(call genTestAppImagePush,$(ITEM))))

Target: push-kind-e2e-app

define genTestAppImageKindPush
.PHONY: push-kind-e2e-app-$(1)
push-kind-e2e-app-$(1): check-e2e-env
	kind load docker-image $(DAPR_TEST_REGISTRY)/e2e-$(1):$(DAPR_TEST_TAG)
endef

# Generate test app image push targets
$(foreach ITEM,$(E2E_TEST_APPS),$(eval $(call genTestAppImageKindPush,$(ITEM))))

Target: e2e-build-deploy-run

e2e-build-deploy-run: create-test-namespace setup-3rd-party build docker-push docker-deploy-k8s setup-test-components build-e2e-app-all push-e2e-app-all test-e2e-all

枚举 e2e 测试target列表

# Enumerate test app build targets
BUILD_E2E_APPS_TARGETS:=$(foreach ITEM,$(E2E_TEST_APPS),build-e2e-app-$(ITEM))
# Enumerate test app push targets
PUSH_E2E_APPS_TARGETS:=$(foreach ITEM,$(E2E_TEST_APPS),push-e2e-app-$(ITEM))
# Enumerate test app push targets
PUSH_KIND_E2E_APPS_TARGETS:=$(foreach ITEM,$(E2E_TEST_APPS),push-kind-e2e-app-$(ITEM))

Target：build-e2e-app-all

# build test app image
build-e2e-app-all: $(BUILD_E2E_APPS_TARGETS)

Target：push-e2e-app-all

# push test app image to the registry
push-e2e-app-all: $(PUSH_E2E_APPS_TARGETS)

Target：push-kind-e2e-app-all

# push test app image to kind cluster
push-kind-e2e-app-all: $(PUSH_KIND_E2E_APPS_TARGETS)

Target: test-e2e-all

# start all e2e tests
test-e2e-all: check-e2e-env test-deps
	# Note: we can set -p 2 to run two tests apps at a time, because today we do not share state between
	# tests. In the future, if we add any tests that modify global state (such as dapr config), we'll 
	# have to be sure and run them after the main test suite, so as not to alter the state of a running
	# test
	# Note2: use env variable DAPR_E2E_TEST to pick one e2e test to run.
     ifeq ($(DAPR_E2E_TEST),)
	DAPR_CONTAINER_LOG_PATH=$(DAPR_CONTAINER_LOG_PATH) GOOS=$(TARGET_OS_LOCAL) DAPR_TEST_NAMESPACE=$(DAPR_TEST_NAMESPACE) DAPR_TEST_TAG=$(DAPR_TEST_TAG) DAPR_TEST_REGISTRY=$(DAPR_TEST_REGISTRY) DAPR_TEST_MINIKUBE_IP=$(MINIKUBE_NODE_IP) gotestsum --jsonfile $(TEST_OUTPUT_FILE_PREFIX)_e2e.json --junitfile $(TEST_OUTPUT_FILE_PREFIX)_e2e.xml --format standard-quiet -- -p 2 -count=1 -v -tags=e2e ./tests/e2e/$(DAPR_E2E_TEST)/...
     else
	for app in $(DAPR_E2E_TEST); do \
		DAPR_CONTAINER_LOG_PATH=$(DAPR_CONTAINER_LOG_PATH) GOOS=$(TARGET_OS_LOCAL) DAPR_TEST_NAMESPACE=$(DAPR_TEST_NAMESPACE) DAPR_TEST_TAG=$(DAPR_TEST_TAG) DAPR_TEST_REGISTRY=$(DAPR_TEST_REGISTRY) DAPR_TEST_MINIKUBE_IP=$(MINIKUBE_NODE_IP) gotestsum --jsonfile $(TEST_OUTPUT_FILE_PREFIX)_e2e.json --junitfile $(TEST_OUTPUT_FILE_PREFIX)_e2e.xml --format standard-quiet -- -p 2 -count=1 -v -tags=e2e ./tests/e2e/$$app/...; \
	done
     endif

5.3 - 性能测试部分

和性能测试相关的make target

性能测试的 target 定义在 dapr_tests.mk：

https://github.com/dapr/dapr/blob/master/tests/dapr_tests.mk

但因为内容比较独立，单独拿出来细看

变量定义

PERF_TEST_APPS 定义了性能测试要用到的app：

# PERFORMANCE test app list
PERF_TEST_APPS=actorfeatures actorjava tester service_invocation_http

PERF_TESTAPP_DIR 是性能测试应用所在的目录：

# PERFORMANCE test app root directory
PERF_TESTAPP_DIR=./tests/apps/perf

PERF_TESTS 是性能测试包含的测试案例，这些测试案例在 tests/perf 下：

# PERFORMANCE tests
PERF_TESTS=actor_timer actor_reminder actor_activation service_invocation_http

Target: build-perf-app-x

define genPerfTestAppImageBuild
.PHONY: build-perf-app-$(1)
build-perf-app-$(1): check-e2e-env
	$(DOCKER) build -f $(PERF_TESTAPP_DIR)/$(1)/$(DOCKERFILE) $(PERF_TESTAPP_DIR)/$(1)/. -t $(DAPR_TEST_REGISTRY)/perf-$(1):$(DAPR_TEST_TAG)
endef

# Generate perf app image build targets
$(foreach ITEM,$(PERF_TEST_APPS),$(eval $(call genPerfTestAppImageBuild,$(ITEM))))

Target: perf-build-deploy-run

perf-build-deploy-run: create-test-namespace setup-3rd-party build docker-push docker-deploy-k8s setup-test-components build-perf-app-all push-perf-app-all test-perf-all

Target: push-perf-app-x

define genPerfAppImagePush
.PHONY: push-perf-app-$(1)
push-perf-app-$(1): check-e2e-env
	$(DOCKER) push $(DAPR_TEST_REGISTRY)/perf-$(1):$(DAPR_TEST_TAG)
endef

# Generate perf app image push targets
$(foreach ITEM,$(PERF_TEST_APPS),$(eval $(call genPerfAppImagePush,$(ITEM))))

Target: push-kind-perf-app-x

define genPerfAppImageKindPush
.PHONY: push-kind-perf-app-$(1)
push-kind-perf-app-$(1): check-e2e-env
	kind load docker-image $(DAPR_TEST_REGISTRY)/perf-$(1):$(DAPR_TEST_TAG)
endef

# Generate perf app image kind push targets
$(foreach ITEM,$(PERF_TEST_APPS),$(eval $(call genPerfAppImageKindPush,$(ITEM))))

枚举性能测试target列表

# Enumerate test app build targets
BUILD_PERF_APPS_TARGETS:=$(foreach ITEM,$(PERF_TEST_APPS),build-perf-app-$(ITEM))
# Enumerate perf app push targets
PUSH_PERF_APPS_TARGETS:=$(foreach ITEM,$(PERF_TEST_APPS),push-perf-app-$(ITEM))
# Enumerate perf app kind push targets
PUSH_KIND_PERF_APPS_TARGETS:=$(foreach ITEM,$(PERF_TEST_APPS),push-kind-perf-app-$(ITEM))

Target：build-perf-app-all

# build perf app image
build-perf-app-all: $(BUILD_PERF_APPS_TARGETS)

Target：push-perf-app-all

# push perf app image to the registry
push-perf-app-all: $(PUSH_PERF_APPS_TARGETS)

Target：push-kind-perf-app-all

# push perf app image to kind cluster
push-kind-perf-app-all: $(PUSH_KIND_PERF_APPS_TARGETS)

Target: test-perf-x

define genPerfTestRun
.PHONY: test-perf-$(1)
test-perf-$(1): check-e2e-env test-deps
	DAPR_CONTAINER_LOG_PATH=$(DAPR_CONTAINER_LOG_PATH) GOOS=$(TARGET_OS_LOCAL) DAPR_TEST_NAMESPACE=$(DAPR_TEST_NAMESPACE) DAPR_TEST_TAG=$(DAPR_TEST_TAG) DAPR_TEST_REGISTRY=$(DAPR_TEST_REGISTRY) DAPR_TEST_MINIKUBE_IP=$(MINIKUBE_NODE_IP) gotestsum --jsonfile $(TEST_OUTPUT_FILE_PREFIX)_perf_$(1).json --junitfile $(TEST_OUTPUT_FILE_PREFIX)_perf_$(1).xml --format standard-quiet -- -timeout 1h -p 1 -count=1 -v -tags=perf ./tests/perf/$(1)/...
	jq -r .Output $(TEST_OUTPUT_FILE_PREFIX)_perf_$(1).json | strings
endef

# Generate perf app image build targets
$(foreach ITEM,$(PERF_TESTS),$(eval $(call genPerfTestRun,$(ITEM))))

TEST_PERF_TARGETS:=$(foreach ITEM,$(PERF_TESTS),test-perf-$(ITEM))

Target: test-perf-all

# start all perf tests
test-perf-all: check-e2e-env test-deps
	DAPR_CONTAINER_LOG_PATH=$(DAPR_CONTAINER_LOG_PATH) GOOS=$(TARGET_OS_LOCAL) DAPR_TEST_NAMESPACE=$(DAPR_TEST_NAMESPACE) DAPR_TEST_TAG=$(DAPR_TEST_TAG) DAPR_TEST_REGISTRY=$(DAPR_TEST_REGISTRY) DAPR_TEST_MINIKUBE_IP=$(MINIKUBE_NODE_IP) gotestsum --jsonfile $(TEST_OUTPUT_FILE_PREFIX)_perf.json --junitfile $(TEST_OUTPUT_FILE_PREFIX)_perf.xml --format standard-quiet -- -p 1 -count=1 -v -tags=perf ./tests/perf/...
	jq -r .Output $(TEST_OUTPUT_FILE_PREFIX)_perf.json | strings

m1上执行

执行结果输出如下：

$ make test-perf-all                  
# The desire here is to download this test dependency without polluting go.mod
# In golang >=1.16 there is a new way to do this with `go install gotest.tools/gotestsum@latest`
# But this doesn't work with <=1.15.
# (see: https://golang.org/ref/mod#go-install)
command -v gotestsum || go install gotest.tools/gotestsum@latest
/Users/sky/work/soft/gopath/bin/gotestsum
DAPR_CONTAINER_LOG_PATH=./dist/container_logs GOOS=darwin DAPR_TEST_NAMESPACE=dapr-system DAPR_TEST_TAG=dev-linux-arm64 DAPR_TEST_REGISTRY=docker.io/skyao DAPR_TEST_MINIKUBE_IP= gotestsum --jsonfile ./test_report_perf.json --junitfile ./test_report_perf.xml --format standard-quiet -- -p 1 -count=1 -v -tags=perf ./tests/perf/...
?       github.com/dapr/dapr/tests/perf [no test files]
2022/03/24 10:52:06 Running setup...
2022/03/24 10:52:06 Installing test apps...
2022/03/24 10:52:06 Adding app {testapp 3000  map[] true perf-actorjava:dev-linux-arm64  docker.io/skyao 1 true true   4.0 0.1 800Mi 2500Mi 4.0 0.1 512Mi 250Mi <nil> false}
2022/03/24 10:52:06 Adding app {tester 3001  map[] true perf-tester:dev-linux-arm64  docker.io/skyao 1 true true   4.0 0.1 800Mi 2500Mi 4.0 0.1 512Mi 250Mi <nil> false}
2022/03/24 10:52:06 Installing apps ...
2022/03/24 10:52:08 Deploying app testapp ...

2022/03/24 11:02:10 deployment testapp relate pods: ......
CST,LastTransitionTime:2022-03-24 11:02:10 +0800 CST,},},ReadyReplicas:0,CollisionCount:nil,},} pod status: map[testapp-7cc8965d47-fzqhs:[]] error: timed out waiting for the condition2022/03/24 11:02:10 Running teardown...
FAIL    github.com/dapr/dapr/tests/perf/actor_activation        605.705s