docker.go
324 lines1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
package platform
import (
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
// StartService runs a service container on the server.
func StartService(s *Server, service *Service) error {
args := []string{"docker", "run", "-d", "--name", service.Name}
// Add restart policy
if service.Restart != "" {
args = append(args, "--restart", service.Restart)
}
// Add network
if service.Network != "" {
args = append(args, "--network", service.Network)
}
// Add ports
for _, p := range service.Ports {
portMap := fmt.Sprintf("%d:%d", p.Host, p.Container)
if p.Bind != "" {
portMap = p.Bind + ":" + portMap
}
args = append(args, "-p", portMap)
}
// Add volumes
for _, v := range service.Volumes {
args = append(args, "-v", v.Source+":"+v.Target)
}
// Add environment variables
for k, v := range service.Env {
args = append(args, "-e", k+"="+v)
}
// Add healthcheck
if h := service.Healthcheck; h != nil {
if h.Cmd != "" {
args = append(args, "--health-cmd", h.Cmd)
}
if h.Interval != "" {
args = append(args, "--health-interval", h.Interval)
}
if h.Timeout != "" {
args = append(args, "--health-timeout", h.Timeout)
}
if h.Retries > 0 {
args = append(args, "--health-retries", fmt.Sprintf("%d", h.Retries))
}
if h.StartPeriod != "" {
args = append(args, "--health-start-period", h.StartPeriod)
}
}
// Add resource limits
if r := service.Resources; r != nil {
if r.CPUs != "" {
args = append(args, "--cpus", r.CPUs)
}
if r.Memory != "" {
args = append(args, "--memory", r.Memory)
}
}
// Add logging configuration
if l := service.Logging; l != nil {
if l.Driver != "" {
args = append(args, "--log-driver", l.Driver)
}
for k, v := range l.Options {
args = append(args, "--log-opt", k+"="+v)
}
}
// Add privileged flag
if service.Privileged {
args = append(args, "--privileged")
}
// Add image
args = append(args, service.Image)
// Add command
args = append(args, service.Command...)
_, err := s.SSH(args...)
return err
}
// StopService kills and removes a container on the server.
func StopService(s *Server, name string) error {
_, err := s.SSH("docker", "rm", "-f", name)
return err
}
// UploadImage uploads a Docker image tar.gz to the server and loads it.
// Returns the image name extracted from the file path.
func UploadImage(s *Server, imagePath string) (string, error) {
if imagePath == "" {
return "", nil
}
log.Printf(" Uploading image...")
base := filepath.Base(imagePath)
imageName := strings.TrimSuffix(base, ".tar.gz")
remotePath := "/tmp/" + base
if err := s.Copy(imagePath, remotePath); err != nil {
return "", err
}
log.Printf(" Loading image...")
if _, err := s.SSHWithTimeout(5*time.Minute, "bash", "-c", fmt.Sprintf("gunzip -c %s | docker load", remotePath)); err != nil {
return "", fmt.Errorf("docker load: %w", err)
}
s.SSH("rm", remotePath)
return imageName, nil
}
// BuildImage uploads source and builds a Docker image on the server.
// Uses git archive to capture all tracked files, avoiding brittle hardcoded lists.
func BuildImage(s *Server, name, source string) (string, error) {
log.Printf(" Building on server...")
source = strings.TrimPrefix(source, "./")
remoteBuildDir := "/tmp/build-" + name
if _, err := s.SSH("rm", "-rf", remoteBuildDir); err != nil {
return "", fmt.Errorf("clean build dir: %w", err)
}
if _, err := s.SSH("mkdir", "-p", remoteBuildDir); err != nil {
return "", fmt.Errorf("create build dir: %w", err)
}
// Warn if there are uncommitted changes (git archive only captures committed content)
if out, err := exec.Command("git", "status", "--porcelain").Output(); err == nil && len(out) > 0 {
log.Printf(" WARNING: uncommitted changes will NOT be included in deploy")
}
// Upload entire project via git archive (respects .gitignore, no build artifacts)
log.Printf(" Uploading project...")
tarPath := "/tmp/build-" + name + ".tar.gz"
gitArchive := exec.Command("git", "archive", "--format=tar.gz", "-o", tarPath, "HEAD")
if out, err := gitArchive.CombinedOutput(); err != nil {
return "", fmt.Errorf("git archive: %s: %w", string(out), err)
}
defer os.Remove(tarPath)
remoteTar := "/tmp/build-" + name + ".tar.gz"
if err := s.Copy(tarPath, remoteTar); err != nil {
return "", fmt.Errorf("upload archive: %w", err)
}
if _, err := s.SSH("tar", "-xzf", remoteTar, "-C", remoteBuildDir); err != nil {
return "", fmt.Errorf("extract archive: %w", err)
}
s.SSH("rm", remoteTar)
// Use source-specific Dockerfile if present (e.g. dns/Dockerfile)
if source != "." {
if srcDockerfile := filepath.Join(source, "Dockerfile"); fileExists(srcDockerfile) {
copyCmd := fmt.Sprintf("cp %s/%s %s/Dockerfile", remoteBuildDir, srcDockerfile, remoteBuildDir)
s.SSH("bash", "-c", copyCmd)
}
}
// Build on server with BuildKit (10 minute timeout — builds can be slow)
log.Printf(" Running docker build...")
buildCmd := fmt.Sprintf("cd %s && DOCKER_BUILDKIT=1 docker build -t %s -f Dockerfile .", remoteBuildDir, name)
if _, err := s.SSHWithTimeout(10*time.Minute, "bash", "-c", buildCmd); err != nil {
return "", fmt.Errorf("remote docker build: %w", err)
}
s.SSH("rm", "-rf", remoteBuildDir)
return name, nil
}
// WaitForHealthy waits for a Docker container to report healthy status.
func WaitForHealthy(s *Server, container string, timeoutSecs int) error {
log.Printf(" Waiting for %s to become healthy...", container)
for i := 0; i < timeoutSecs; i += 5 {
status, err := s.SSH("docker", "inspect", "--format={{.State.Health.Status}}", container)
if err == nil {
status = strings.TrimSpace(status)
switch status {
case "healthy":
log.Printf(" %s is healthy", container)
return nil
case "unhealthy":
logs, _ := s.SSH("docker", "logs", "--tail=20", container)
return fmt.Errorf("container unhealthy, logs:\n%s", logs)
}
}
time.Sleep(5 * time.Second)
}
return fmt.Errorf("health check timeout after %ds", timeoutSecs)
}
// BackupContainer stops and renames the current container for rollback.
func BackupContainer(s *Server, name string) {
backupName := name + "-backup"
s.SSH("docker", "rm", "-f", backupName)
s.SSH("docker", "stop", name)
s.SSH("docker", "rename", name, backupName)
}
// Rollback restores the previous container version from backup.
func Rollback(s *Server, name string) error {
backupName := name + "-backup"
if _, err := s.SSH("docker", "inspect", backupName); err != nil {
log.Printf(" No backup found for %s, cannot rollback", name)
return nil
}
log.Printf(" Rolling back %s...", name)
s.SSH("docker", "rm", "-f", name)
if _, err := s.SSH("docker", "rename", backupName, name); err != nil {
return fmt.Errorf("rename backup: %w", err)
}
if _, err := s.SSH("docker", "start", name); err != nil {
return fmt.Errorf("start backup: %w", err)
}
log.Printf(" Rolled back to previous version of %s", name)
return nil
}
// RemoveContainer force-removes a Docker container by name.
func RemoveContainer(s *Server, name string) {
s.SSH("docker", "rm", "-f", name)
}
// ConfigureInsecureRegistry writes /etc/docker/daemon.json on the server
// to allow pulling from an insecure (HTTP) private registry.
// Only restarts Docker if the config actually changed.
func ConfigureInsecureRegistry(s *Server, addr string) {
log.Printf(" Configuring insecure registry %s on %s...", addr, s.Name)
daemonJSON := fmt.Sprintf(`{"insecure-registries":["%s"]}`, addr)
if existing, _ := s.SSH("cat", "/etc/docker/daemon.json"); strings.TrimSpace(existing) == daemonJSON {
log.Printf(" Insecure registry already configured on %s", s.Name)
return
}
s.Write("/etc/docker/daemon.json", []byte(daemonJSON), false)
script := `systemctl restart docker; for i in $(seq 1 30); do docker info >/dev/null 2>&1 && break; sleep 1; done`
if _, err := s.SSH("bash", "-c", script); err != nil {
log.Printf(" Warning: failed to configure insecure registry on %s: %v", s.Name, err)
}
}
// WaitForRegistry waits for a Docker registry to be reachable from this server.
func WaitForRegistry(s *Server, host string) {
script := fmt.Sprintf(`
for i in $(seq 1 30); do
curl -sf http://%s/v2/ >/dev/null 2>&1 && exit 0
sleep 2
done
echo "Registry not ready after 60s"
exit 1
`, host)
if _, err := s.SSH("bash", "-c", script); err != nil {
log.Printf(" Warning: registry may not be ready: %v", err)
}
}
// SyncSecrets copies /etc/secrets/* from one server to each target server.
// Public keys and SSH keys are set to 644 so non-root container users can read them.
func SyncSecrets(from *Server, to []*Server) {
listing, err := from.SSH("ls", "/etc/secrets/")
if err != nil {
log.Printf(" Warning: could not list secrets on manager: %v", err)
return
}
files := strings.Fields(strings.TrimSpace(listing))
if len(files) == 0 {
return
}
for _, worker := range to {
log.Printf(" Syncing secrets to %s...", worker.Name)
worker.SSH("mkdir", "-p", "/etc/secrets")
for _, filename := range files {
// Use base64 to safely transfer binary/multi-line files (e.g. SSH keys)
b64, err := from.SSH("base64", "-w0", "/etc/secrets/"+filename)
if err != nil {
log.Printf(" Warning: could not read %s from manager: %v", filename, err)
continue
}
b64 = strings.TrimSpace(b64)
if b64 == "" {
continue
}
// SSH keys and public keys need 644 for non-root container access
mode := "600"
if strings.HasSuffix(filename, ".pub") || strings.HasSuffix(filename, "_public_key.pem") || filename == "ssh_key" {
mode = "644"
}
script := fmt.Sprintf("echo %s | base64 -d > /etc/secrets/%s && chmod %s /etc/secrets/%s",
shellEscape(b64), shellEscape(filename), mode, shellEscape(filename))
worker.SSH("bash", "-c", script)
}
}
}