diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b6e340..1c9e375 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,41 +39,34 @@ yaml-templates: root: . paths: - dist + +orbs: + gke: circleci/gcp-gke@x.y.z + workflows: version: 2 build-workflow: jobs: - build-linux-386: <<: *release_filters - - build-linux-amd64: - <<: *branch_filters - build-darwin-386: - <<: *release_filters - - build-darwin-amd64: - <<: *branch_filters + <<: *release_filters - build-windows-386: <<: *release_filters - - build-windows-amd64: - <<: *branch_filters - build-linux-arm: <<: *release_filters - build-linux-arm64: <<: *release_filters - test: <<: *branch_filters + - smoke_test: + <<: *branch_filters - publish-github-release: <<: *release_filters requires: - - build-linux-386 - - build-linux-amd64 - - build-darwin-386 - - build-darwin-amd64 - - build-windows-386 - - build-windows-amd64 - - build-linux-arm - - build-linux-arm64 - test + - smoke_test jobs: build-linux-amd64: &go_build @@ -153,6 +146,56 @@ jobs: name: Run Tests command: go test ./... + smoke_test: + machine: true + environment: &environment + steps: + - run: + name: export + command: | + echo 'export KUBECONFIG=/home/circleci/.kube/config' >> $BASH_ENV + echo 'export GOROOT=/usr/local/go' >> $BASH_ENV + echo 'export GOPATH=$HOME/Projects/Proj1' >> $BASH_ENV + echo 'export PATH=$GOPATH/bin:$GOROOT/bin:$PATH' >> $BASH_ENV + source $BASH_ENV + - checkout + - run: + name: print environment + command: printenv + - run: + name: uname + command: uname -a + - run: + name: print go version + command: go version + - run: + name: install go 1.13 + command: | + wget https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz + sudo tar -xvf go1.13.3.linux-amd64.tar.gz + sudo rm -rf /usr/local/go && sudo mv go /usr/local + - run: + name: print go version + command: go version + - run: + name: get kind + command: env GO111MODULE=off go get sigs.k8s.io/kind + - run: + name: compile + command: make + - run: + name: Create kind clusterr + command: kind create cluster --wait 30m #bla + - run: + name: trying to install kubectl + command: curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.17.4/bin/linux/amd64/kubectl && mkdir -p ~/bin && mv kubectl ~/bin/ && chmod +x ~/bin/kubectl + - run: + name: print cluster info + command: kubectl cluster-info + - run: + name: Run Tests + command: ./skupper_smoke #set kubeconfig as parameter if needed + publish-github-release: docker: - image: cibuilds/github:0.10 diff --git a/Makefile b/Makefile index fa78bf7..06180bb 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ all: build build: go build -ldflags="-X main.version=${VERSION}" -o skupper cmd/skupper/skupper.go + go build -ldflags="-X main.version=${VERSION}" -o skupper_smoke cmd/skupper_smoke/skupper_smoke.go clean: rm -rf skupper release diff --git a/cmd/skupper_smoke/skupper_smoke.go b/cmd/skupper_smoke/skupper_smoke.go new file mode 100644 index 0000000..c7c027a --- /dev/null +++ b/cmd/skupper_smoke/skupper_smoke.go @@ -0,0 +1,29 @@ +package main + +import ( + "flag" + "path/filepath" + + "k8s.io/client-go/util/homedir" + + "github.com/skupperproject/skupper-cli/pkg/smoke" + "github.com/skupperproject/skupper-cli/pkg/smoke/tcp_echo" +) + +func main() { + testRunners := []smoke.SmokeTestRunnerInterface{&tcp_echo.SmokeTestRunner{}} + + defaultKubeConfig := filepath.Join(homedir.HomeDir(), ".kube", "config") + + pub1Kubeconfig := flag.String("pub1kubeconfig", defaultKubeConfig, "(optional) absolute path to the kubeconfig file") + pub2Kubeconfig := flag.String("pub2kubeconfig", defaultKubeConfig, "(optional) absolute path to the kubeconfig file") + priv1Kubeconfig := flag.String("priv1kubeconfig", defaultKubeConfig, "(optional) absolute path to the kubeconfig file") + priv2Kubeconfig := flag.String("priv2kubeconfig", defaultKubeConfig, "(optional) absolute path to the kubeconfig file") + + flag.Parse() + + for _, testRunner := range testRunners { + testRunner.Build(*pub1Kubeconfig, *pub2Kubeconfig, *priv1Kubeconfig, *priv2Kubeconfig) + testRunner.Run() + } +} diff --git a/pkg/smoke/smoke_test_runner.go b/pkg/smoke/smoke_test_runner.go new file mode 100644 index 0000000..ba8e9f9 --- /dev/null +++ b/pkg/smoke/smoke_test_runner.go @@ -0,0 +1,128 @@ +package smoke + +import ( + "fmt" + "log" + "os/exec" + "time" + + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +type SmokeTestRunnerInterface interface { + Build(public1ConficFile, public2ConficFile, private1ConfigFile, private2ConfigFile string) + Run() +} + +type SmokeTestRunnerBase struct { + Pub1Cluster *ClusterContext + Pub2Cluster *ClusterContext + Priv1Cluster *ClusterContext + Priv2Cluster *ClusterContext +} + +func (r *SmokeTestRunnerBase) Build(public1ConficFile, public2ConficFile, private1ConfigFile, private2ConfigFile string) { + r.Pub1Cluster = BuildClusterContext("public1", public1ConficFile) + r.Pub2Cluster = BuildClusterContext("public2", public2ConficFile) + r.Priv1Cluster = BuildClusterContext("private1", private1ConfigFile) + r.Priv2Cluster = BuildClusterContext("private2", private2ConfigFile) +} + +type ClusterContext struct { + Namespace string + ClusterConfigFile string + Clientset *kubernetes.Clientset +} + +func BuildClusterContext(namespace string, configFile string) *ClusterContext { + cc := &ClusterContext{} + cc.Namespace = namespace + cc.ClusterConfigFile = configFile + + config, err := clientcmd.BuildConfigFromFlags("", configFile) + if err != nil { + log.Panic(err.Error()) + } + + cc.Clientset, err = kubernetes.NewForConfig(config) + if err != nil { + log.Panic(err.Error()) + } + return cc +} + +func _exec(command string, wait bool) { + var output []byte + var err error + fmt.Println(command) + cmd := exec.Command("sh", "-c", command) + if wait { + output, err = cmd.CombinedOutput() + } else { + cmd.Start() + return + } + if err != nil { + panic(err) + } + fmt.Println(string(output)) +} + +func (cc *ClusterContext) exec(main_command string, sub_command string, wait bool) { + _exec("KUBECONFIG="+cc.ClusterConfigFile+" "+main_command+" "+cc.Namespace+" "+sub_command, wait) +} + +func (cc *ClusterContext) SkupperExec(command string) { + cc.exec("./skupper -n ", command, true) +} + +func (cc *ClusterContext) _kubectl_exec(command string, wait bool) { + cc.exec("kubectl -n ", command, wait) +} + +func (cc *ClusterContext) KubectlExec(command string) { + cc._kubectl_exec(command, true) +} + +func (cc *ClusterContext) KubectlExecAsync(command string) { + cc._kubectl_exec(command, false) +} + +func (cc *ClusterContext) CreateNamespace() { + NsSpec := &apiv1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: cc.Namespace}} + _, err := cc.Clientset.CoreV1().Namespaces().Create(NsSpec) + if err != nil { + log.Panic(err.Error()) + } +} + +func (cc *ClusterContext) DeleteNamespace() { + err := cc.Clientset.CoreV1().Namespaces().Delete(cc.Namespace, &metav1.DeleteOptions{}) + if err != nil { + log.Panic(err.Error()) + } +} + +func (cc *ClusterContext) GetService(name string, timeout_S time.Duration) *apiv1.Service { + timeout := time.After(timeout_S * time.Second) + tick := time.Tick(3 * time.Second) + for { + select { + case <-timeout: + log.Panicln("Timed Out Waiting for service.") + case <-tick: + service, err := cc.Clientset.CoreV1().Services(cc.Namespace).Get(name, metav1.GetOptions{}) + if err == nil { + return service + } else { + log.Println("Service not ready yet, current pods state: ") + cc.KubectlExec("get pods -o wide") //TODO use clientset + } + + } + } +} diff --git a/pkg/smoke/tcp_echo/tcp_echo.go b/pkg/smoke/tcp_echo/tcp_echo.go new file mode 100644 index 0000000..bf27ab6 --- /dev/null +++ b/pkg/smoke/tcp_echo/tcp_echo.go @@ -0,0 +1,182 @@ +package tcp_echo + +import ( + "fmt" + "log" + "net" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/skupperproject/skupper-cli/pkg/smoke" +) + +type SmokeTestRunner struct { + smoke.SmokeTestRunnerBase +} + +func int32Ptr(i int32) *int32 { return &i } + +const minute time.Duration = 60 + +var deployment *appsv1.Deployment = &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-go-echo", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"application": "tcp-go-echo"}, + }, + Template: apiv1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "application": "tcp-go-echo", + }, + }, + Spec: apiv1.PodSpec{ + Containers: []apiv1.Container{ + { + Name: "tcp-go-echo", + Image: "quay.io/skupper/tcp-go-echo", + ImagePullPolicy: apiv1.PullIfNotPresent, + Ports: []apiv1.ContainerPort{ + { + Name: "http", + Protocol: apiv1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, +} + +func sendReceive(servAddr string) { + strEcho := "Halo" + //servAddr := ip + ":9090" + tcpAddr, err := net.ResolveTCPAddr("tcp", servAddr) + if err != nil { + log.Panicln("ResolveTCPAddr failed:", err.Error()) + } + conn, err := net.DialTCP("tcp", nil, tcpAddr) + if err != nil { + log.Panicln("Dial failed:", err.Error()) + } + _, err = conn.Write([]byte(strEcho)) + if err != nil { + log.Panicln("Write to server failed:", err.Error()) + } + + reply := make([]byte, 1024) + + _, err = conn.Read(reply) + if err != nil { + log.Panicln("Write to server failed:", err.Error()) + } + conn.Close() + + log.Println("Sent to server = ", strEcho) + log.Println("Reply from server = ", string(reply)) + + if !strings.Contains(string(reply), strings.ToUpper(strEcho)) { + log.Panicf("Response from server different that expected: %s", string(reply)) + } +} + +func (r *SmokeTestRunner) RunTests() { + var publicService *apiv1.Service + var privateService *apiv1.Service + + //TODO deduplicate + r.Pub1Cluster.KubectlExec("get svc") + r.Priv1Cluster.KubectlExec("get svc") + + publicService = r.Pub1Cluster.GetService("tcp-go-echo", minute) + privateService = r.Priv1Cluster.GetService("tcp-go-echo", minute) + + fmt.Printf("Public service ClusterIp = %q\n", publicService.Spec.ClusterIP) + fmt.Printf("Private service ClusterIp = %q\n", privateService.Spec.ClusterIP) + + r.Pub1Cluster.KubectlExecAsync("port-forward service/tcp-go-echo 9090:9090") + r.Priv1Cluster.KubectlExecAsync("port-forward service/tcp-go-echo 9091:9090") + + time.Sleep(2 * time.Second) //give time to port forwarding to start + + //sendReceive(publicService.Spec.ClusterIP + ":9090") + //sendReceive(privateService.Spec.ClusterIP + ":9090") + sendReceive("127.0.0.1:9090") + sendReceive("127.0.0.1:9091") +} + +func (r *SmokeTestRunner) Setup() { + r.Pub1Cluster.CreateNamespace() + r.Priv1Cluster.CreateNamespace() + + publicDeploymentsClient := r.Pub1Cluster.Clientset.AppsV1().Deployments(r.Pub1Cluster.Namespace) + + fmt.Println("Creating deployment...") + result, err := publicDeploymentsClient.Create(deployment) + if err != nil { + panic(err) + } + log.Println("================") + //log.Println(err.Error()) + + fmt.Printf("Created deployment %q.\n", result.GetObjectMeta().GetName()) + + fmt.Printf("Listing deployments in namespace %q:\n", "public") + list, err := publicDeploymentsClient.List(metav1.ListOptions{}) + if err != nil { + log.Panic(err.Error()) + } + for _, d := range list.Items { + fmt.Printf(" * %s (%d replicas)\n", d.Name, *d.Spec.Replicas) + } + + r.Pub1Cluster.SkupperExec("init --cluster-local") + r.Pub1Cluster.SkupperExec("expose --port 9090 deployment tcp-go-echo") + r.Pub1Cluster.SkupperExec("connection-token /tmp/public_secret.yaml") + + r.Priv1Cluster.SkupperExec("init --cluster-local") + r.Priv1Cluster.SkupperExec("connect /tmp/public_secret.yaml") + + r.Pub1Cluster.GetService("tcp-go-echo", 10*minute) + r.Priv1Cluster.GetService("tcp-go-echo", 3*minute) +} + +func (r *SmokeTestRunner) TearDown() { + //since this is going to run in a spawned ci vm (then destroyed) probably + //tearDown is not so important + publicDeploymentsClient := r.Pub1Cluster.Clientset.AppsV1().Deployments(r.Pub1Cluster.Namespace) + fmt.Println("Deleting deployment...") + deletePolicy := metav1.DeletePropagationForeground + if err := publicDeploymentsClient.Delete("tcp-go-echo", &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + log.Panic(err.Error()) + } + fmt.Println("Deleted deployment.") + + r.Pub1Cluster.SkupperExec("delete") + r.Priv1Cluster.SkupperExec("delete") + + //r.deleteNamespaces()?? + r.Pub1Cluster.DeleteNamespace() + r.Priv1Cluster.DeleteNamespace() +} + +func (r *SmokeTestRunner) Run() { + defer r.TearDown() + r.Setup() + r.RunTests() +}