From 4d22675b2bfc6488f482006b9847d053c2489227 Mon Sep 17 00:00:00 2001
From: Danny McCormick <dannymccormick@google.com>
Date: Fri, 30 Jun 2023 08:46:53 -0400
Subject: [PATCH] Add flag to do a docker system prune between docker push
 steps (#26967)

* Add flag to do a docker system prune between docker push steps

* Push to my docker image

* order/prune python container pushes

* Prune docker between pushes

* Consolidate to loop

* Temporarily point to my dockerhub to test/confirm

* restore apache as dockerhub repo root

* whitespace
---
 build.gradle.kts                   | 53 ++++++++++++++++++++++++++++--
 sdks/python/container/build.gradle | 30 ++++++++++++++---
 2 files changed, 76 insertions(+), 7 deletions(-)

diff --git a/build.gradle.kts b/build.gradle.kts
index 6ba7ebeaca4..57438fa6768 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -573,18 +573,65 @@ tasks.register("typescriptPreCommit") {
   dependsOn(":sdks:python:test-suites:tox:py38:jest")
 }
 
-tasks.register("pushAllDockerImages") {
+tasks.register("pushAllRunnersDockerImages") {
   dependsOn(":runners:spark:3:job-server:container:dockerPush")
+  for (version in project.ext.get("allFlinkVersions") as Array<*>) {
+    dependsOn(":runners:flink:${version}:job-server-container:dockerPush")
+  }
+  
+  doLast {
+    if (project.hasProperty("prune-images")) {
+      exec {
+        executable("docker")
+        args("system", "prune", "-a", "--force")
+      }
+    }
+  }
+}
+
+tasks.register("pushAllSdkDockerImages") {
+  // Enforce ordering to allow the prune step to happen between runs.
+  // This will ensure we don't use up too much space (especially in CI environments)
+  mustRunAfter(":pushAllRunnersDockerImages")
+
   dependsOn(":sdks:java:container:pushAll")
   dependsOn(":sdks:python:container:pushAll")
   dependsOn(":sdks:go:container:pushAll")
   dependsOn(":sdks:typescript:container:pushAll")
-  for (version in project.ext.get("allFlinkVersions") as Array<*>) {
-    dependsOn(":runners:flink:${version}:job-server-container:dockerPush")
+  
+  doLast {
+    if (project.hasProperty("prune-images")) {
+      exec {
+        executable("docker")
+        args("system", "prune", "-a", "--force")
+      }
+    }
   }
+}
+
+tasks.register("pushAllXlangDockerImages") {
+  // Enforce ordering to allow the prune step to happen between runs.
+  // This will ensure we don't use up too much space (especially in CI environments)
+  mustRunAfter(":pushAllSdkDockerImages")
+
   dependsOn(":sdks:java:expansion-service:container:dockerPush")
   dependsOn(":sdks:java:transform-service:controller-container:dockerPush")
   dependsOn(":sdks:python:expansion-service-container:dockerPush")
+  
+  doLast {
+    if (project.hasProperty("prune-images")) {
+      exec {
+        executable("docker")
+        args("system", "prune", "-a", "--force")
+      }
+    }
+  }
+}
+
+tasks.register("pushAllDockerImages") {
+  dependsOn(":pushAllRunnersDockerImages")
+  dependsOn(":pushAllSdkDockerImages")
+  dependsOn(":pushAllXlangDockerImages")
 }
 
 // Use this task to validate the environment set up for Go, Python and Java
diff --git a/sdks/python/container/build.gradle b/sdks/python/container/build.gradle
index 96a9c17b9a2..f6b31eec5b9 100644
--- a/sdks/python/container/build.gradle
+++ b/sdks/python/container/build.gradle
@@ -42,11 +42,33 @@ tasks.register("buildAll") {
   dependsOn ':sdks:python:container:py311:docker'
 }
 
+for(int i=8; i<=11; ++i) {
+  String cur = "3" + i
+  String prev = "3" + (i-1)
+  tasks.register("push" + cur) {
+    if (cur != "38") {
+      // Enforce ordering to allow the prune step to happen between runs.
+      // This will ensure we don't use up too much space (especially in CI environments)
+      mustRunAfter(":sdks:python:container:push" + prev)
+    }
+    dependsOn ':sdks:python:container:py' + cur + ':dockerPush'
+
+    doLast {
+      if (project.hasProperty("prune-images")) {
+        exec {
+          executable("docker")
+          args("system", "prune", "-a", "--force")
+        }
+      }
+    }
+  }
+}
+
 tasks.register("pushAll") {
-  dependsOn ':sdks:python:container:py38:dockerPush'
-  dependsOn ':sdks:python:container:py39:dockerPush'
-  dependsOn ':sdks:python:container:py310:dockerPush'
-  dependsOn ':sdks:python:container:py311:dockerPush'
+  dependsOn ':sdks:python:container:push38'
+  dependsOn ':sdks:python:container:push39'
+  dependsOn ':sdks:python:container:push310'
+  dependsOn ':sdks:python:container:push311'
 }
 
 tasks.register("generatePythonRequirementsAll") {
-- 
GitLab