init repository

This commit is contained in:
rmanach 2023-08-05 22:10:48 +02:00
commit d9ce80beb1
25 changed files with 2016 additions and 0 deletions

1
.env.example Normal file
View File

@ -0,0 +1 @@
# all the environment variables needed for your microservices should go here

5
.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
builds
.env
nginx/ssl/*.csr
nginx/ssl/*.crt
nginx/ssl/private.key

134
.golangci.yml Normal file
View File

@ -0,0 +1,134 @@
linters-settings:
# depguard: // Specific for golangci repository
# list-type: blacklist
# packages:
# # logging is allowed only by logutils.Log, logrus
# # is allowed to use only in logutils package
# - github.com/sirupsen/logrus
# packages-with-error-message:
# - github.com/sirupsen/logrus: 'logging is allowed only by logutils.Log'
dupl:
threshold: 100
funlen:
lines: 100
statements: 50
gci:
local-prefixes: localenv
goconst:
min-len: 2
min-occurrences: 2
gocritic:
enabled-tags:
- diagnostic
- experimental
- opinionated
- performance
- style
disabled-checks:
- dupImport # https://github.com/go-critic/go-critic/issues/845
- ifElseChain
- octalLiteral
# - whyNoLint
- wrapperFunc
gocyclo:
min-complexity: 15
goimports:
local-prefixes: localenv
gomnd:
settings:
mnd:
# don't include the "operation" and "assign"
checks:
- argument
- case
- condition
- return
govet:
check-shadowing: true
# settings: // Specific for golangci repository
# printf:
# funcs:
# - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
# - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
# - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
# - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
lll:
line-length: 200
maligned:
suggest-new: true
misspell:
locale: US
nolintlint:
allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
allow-unused: false # report any unused nolint directives
require-explanation: false # don't require an explanation for nolint directives
require-specific: false # don't require nolint directives to be specific about which linter is being skipped
errcheck:
check-blank: true
exclude-functions:
- '(*github.com/gin-gonic/gin.Error).SetType'
- '(*github.com/gin-gonic/gin.Context).Error'
linters:
disable-all: true
enable:
- bodyclose
# - deadcode # deprecated (since v1.49.0)
# - depguard
- dogsled
- dupl
- errcheck
- exportloopref
- exhaustive
- funlen
- gochecknoinits
- goconst
- gocritic
- gocyclo
- gofmt
- goimports
- gomnd
- goprintffuncname
- gosec
- gosimple
- govet
- ineffassign
- lll
- misspell
- nakedret
- noctx
- nolintlint
# - rowserrcheck # https://github.com/golangci/golangci-lint/issues/2649
- staticcheck
# - structcheck # https://github.com/golangci/golangci-lint/issues/2649
- stylecheck
- typecheck
- unconvert
- unparam
- unused
# - varcheck # deprecated (since v1.49.0)
- whitespace
# - gochecknoglobals # too many global in ds9
# don't enable:
# - asciicheck
# - scopelint
# - gocognit
# - godot
# - godox
# - goerr113
# - interfacer
# - maligned
# - nestif
# - prealloc
# - testpackage
# - revive
# - wsl
# issues:
# Excluding configuration per-path, per-linter, per-text and per-source
# fix: true
run:
timeout: 5m
skip-dirs: []

42
Makefile Normal file
View File

@ -0,0 +1,42 @@
.DEFAULT_GOAL := run
BIN_NAME := localenv
ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
BUILD_DIR := $(ROOT_DIR)builds
build: format lint
@mkdir -p $(BUILD_DIR)
@go build -o $(BUILD_DIR)/$(BIN_NAME) main.go
format:
go fmt ./...
lint:
golangci-lint run --fix
test:
go test ./...
start: build
@mkdir -p /tmp/localenv-swarm
@$(BUILD_DIR)/$(BIN_NAME)
start-watch: build
@mkdir -p /tmp/localenv-swarm
@$(BUILD_DIR)/$(BIN_NAME) --watch
stop:
docker stop localenv-swarm
purge:
docker rm -fv localenv-swarm
enter:
docker exec -it localenv-swarm /bin/bash
docker-swarm:
docker build . -t localenv-swarm:latest -f swarm/Dockerfile
docker-nginx:
@cd nginx/ssl && ./generate-certs.bash
docker build . -t localenv-nginx:latest -f nginx/Dockerfile

85
README.md Normal file
View File

@ -0,0 +1,85 @@
# localenv
This project aims to deploy a microservices environment on local. It will connect and deploy all the microservices and their dependencies (postgres, rabbitmq, etc...).
## Requirements
* Linux (feel sorry for macos users 😭)
* dockerd >= 20.10.23
* [sysbox](https://github.com/nestybox/sysbox/blob/master/docs/user-guide/install-package.md)
* go >= 1.20
* dockerd ubuntu image
```bash
make docker-swarm
```
* nginx with embedded microservice conf
```bash
make docker-nginx
```
## Run
* Create a `.env` and feed values for your needs
```bash
cp .env.example .env
```
* Launch/install the local environment
```bash
make start
```
* Stop it
```bash
make stop
```
* Purge (remove the environment and all the data)
> Be careful ! All the local data will be removed !
```bash
make purge
```
* Enable watcher
> The watcher is used to auto deploy a microservice on the Swarm when the microservice is built.
```bash
make start-watch
```
## How it works ?
`localenv` uses a container with **Docker Swarm** to deploy microservices as Docker services. It is completely isolated and does not pollute your docker environment.
```ascii
local dockerd
┌─────────────────────────────────────────────────────────────────────┐
│ docker swarm (DinD) │
│ ┌───────────────────────────────────────────────────────────────┐ │
│ │ │ │
│ │ ┌────────────┐ ┌───────────────────────────────┐ │ │
│ │ │rabbitmq │ │ │ │ │
│ │ │ │ │ ┌─────────┐ ┌─────────┐ │ │ │
│ │ │ ├────────┼─┤ms │ │postgres │ │ │ │
│ │ │ │ │ │ ├──────┤ │ │ │ │
│ │ └──┬─────────┘ │ │ │ │ │ │ │ │
│ │ │:15672 │ │ │ │ │ │ │ │
│ │ │ │ └─┬───────┘ └────┬────┘ │ │ │
│ │ │ │ │ │ │ │ │
│ │ │ ┌──────────┐ │ │ ┌───────────────┴─────┐ │ │ │
│ │ │ │nginx │ │ │ │persistent storage │ │ │ │
│ │ │ │ ├───┼───┘ └─────────────────────┘ │ │ │
│ │ │ │ │ │ │ │ │
│ │ │ └──┬───────┘ │ db-network│ │ │
│ │ │ │:4443 └───────────────────────────────┘ │ │
│ │ │ │ │ │
│ │ │ │ │ │
│ │ │ │ │:4523 ms-network│ │
│ └────┼──────┼───────────────────────────────┼───────────────────┘ │
│ │ │ │ │
└───────┼──────┼───────────────────────────────┼──────────────────────┘
│ │ │
│ │ │
▼ ▼ ▼
```
The scheme above explains how the `localenv `is deployed.
* A container, running with the **sysbox** runtime, as a Docker Swarm.
* The Swarm **dockerd** is available from the outside to interact easily with it.
* `ms` is connected to the `ms-network` to communicate with the others microservices (DNS resolution).
* A Postgres DB can be connected to a microservice and be isolated in a network. For simplicity, if a microservice needs a database, a new database is deployed.
* **RabbitMQ** is common between all microservices. The management UI is available and accessible at http://localhost:15672.
* **nginx** as a reverse proxy. To access `authorizations` for example: https://localhost:4443/authorizations/health.

64
client/client.go Normal file
View File

@ -0,0 +1,64 @@
package client
import (
"context"
"errors"
"time"
"github.com/docker/docker/client"
"github.com/rs/zerolog/log"
)
const (
WaitConnectionFailed time.Duration = 5 * time.Second
DockerSwarmHostURL string = "tcp://127.0.0.1:4523"
DockerHostURL string = "unix:///var/run/docker.sock"
ClientConnectionAttempts int = 4
)
var ErrSwarmClientError = errors.New("unable to connect to the swarm")
func GetSwarmClient() (*client.Client, error) {
return initClientWithRetry(DockerSwarmHostURL, ClientConnectionAttempts)
}
func GetHostClient() (*client.Client, error) {
return initClientWithRetry(DockerHostURL, ClientConnectionAttempts)
}
func initClient(url string) (*client.Client, error) {
cli, err := client.NewClientWithOpts(
client.WithHost(url),
client.WithAPIVersionNegotiation(),
)
if err != nil {
return cli, err
}
return cli, nil
}
func initClientWithRetry(url string, attempt int) (*client.Client, error) {
var i int
for {
if i == attempt {
return nil, ErrSwarmClientError
}
cli, err := initClient(url)
if err != nil {
return nil, err
}
if _, err := cli.Ping(context.TODO()); err != nil {
log.Error().Str("url", url).Msg("unable to connect to the swarm, retrying...")
time.Sleep(WaitConnectionFailed)
i++
continue
}
return cli, nil
}
}

189
collector/collector.go Normal file
View File

@ -0,0 +1,189 @@
package collector
import (
"context"
"errors"
"io"
"os"
"path"
"strings"
"sync"
"time"
"localenv/utils"
"github.com/docker/docker/client"
"github.com/rs/zerolog/log"
)
const HostImagesStore string = "/tmp/localenv-swarm"
var ErrImageNotFound = errors.New("image not found")
func imageNameIntoFilename(name string) string {
name = strings.Replace(name, "/", "-", -1)
return strings.Replace(name, ":", "-", -1)
}
type collectorErr struct {
errs []error
sync.Mutex
}
func newCollectorError() collectorErr {
return collectorErr{
errs: []error{},
}
}
func (c *collectorErr) AddError(err error) {
c.Lock()
defer c.Unlock()
c.errs = append(c.errs, err)
}
func (c *collectorErr) Err() error {
if len(c.errs) != 0 {
return errors.Join(c.errs...)
}
return nil
}
type Collector struct {
hostCLI *client.Client
swarmCLI *client.Client
}
func NewCollector(hostCLI, swarmCLI *client.Client) Collector {
return Collector{
hostCLI: hostCLI,
swarmCLI: swarmCLI,
}
}
func (c Collector) saveImage(ctx context.Context, name, filepath string) error {
log.Info().Str("image", name).Str("path", filepath).Msg("saving image into store...")
imageSaveResponse, err := c.hostCLI.ImageSave(ctx, []string{name})
if err != nil {
return err
}
defer imageSaveResponse.Close()
tarFile, err := os.Create(filepath)
if err != nil {
return err
}
defer tarFile.Close()
content, err := io.ReadAll(imageSaveResponse)
if err != nil {
return err
}
if _, err := tarFile.Write(content); err != nil {
return err
}
log.Info().Str("image", name).Str("path", filepath).Msg("image saved successfully")
return nil
}
func (c Collector) loadImage(ctx context.Context, filepath string) error {
log.Info().Str("path", filepath).Msg("loading image...")
tarFile, err := os.Open(filepath)
if err != nil {
return err
}
defer tarFile.Close()
loadResponse, err := c.swarmCLI.ImageLoad(ctx, tarFile, false)
if err != nil {
return err
}
defer loadResponse.Body.Close()
log.Info().Str("path", filepath).Msg("image loaded successfully")
return nil
}
func (c Collector) retryCheckImage(ctx context.Context, name string) error {
fnRetry := func(ctx context.Context) error {
ok, err := c.checkImage(ctx, name)
if err != nil {
log.Warn().Str("image", name).Msg("error while searching image, retrying...")
return err
}
if !ok {
log.Warn().Str("image", name).Msg("image not found, retrying...")
return ErrImageNotFound
}
return nil
}
attempts := 30
waitDuration := 2 * time.Second
if err := utils.Retry(ctx, fnRetry, waitDuration, attempts); err != nil {
return err
}
return nil
}
func (c Collector) checkImage(ctx context.Context, name string) (bool, error) {
images, err := utils.FilterImagesByName(ctx, c.swarmCLI, name)
if err != nil {
return false, err
}
if len(images) != 1 {
return false, nil
}
return true, nil
}
func (c Collector) DeployImages(ctx context.Context, images []string) error {
errs := newCollectorError()
var wg sync.WaitGroup
wg.Add(len(images))
for i := range images {
go func(idx int) {
defer wg.Done()
ok, err := c.checkImage(ctx, images[idx])
if err != nil {
errs.AddError(err)
}
if ok {
return
}
filepath := path.Join(HostImagesStore, imageNameIntoFilename(images[idx])+".tar")
if err = c.saveImage(ctx, images[idx], filepath); err != nil {
errs.AddError(err)
}
if err = c.loadImage(ctx, filepath); err != nil {
errs.AddError(err)
}
if err := c.retryCheckImage(ctx, images[idx]); err != nil {
errs.AddError(err)
}
}(i)
}
wg.Wait()
return errs.Err()
}

211
deployer/deployer.go Normal file
View File

@ -0,0 +1,211 @@
package deployer
import (
"context"
"errors"
"strings"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client"
"github.com/docker/go-connections/nat"
"github.com/rs/zerolog/log"
localcli "localenv/client"
"localenv/services"
)
const (
SwarmImageName string = "localenv-swarm:latest"
SwarmContainerName string = "localenv-swarm"
)
var (
ErrSwarmNotFound = errors.New("unable to find the swarm")
ErrSwarmClientInit = errors.New("swarm client is not initialized")
ImagesDeps = []string{
services.PostgresImageName,
services.RabbitMQImageName,
services.NginxImageName,
services.MailhogImageName,
}
)
type Deployer struct {
swarm Swarm
cli *client.Client
}
func NewDeployer(ctx context.Context, cli *client.Client) (Deployer, error) {
var deployer Deployer
deployer.cli = cli
if err := deployer.init(ctx); err != nil {
return deployer, err
}
return deployer, nil
}
func (d Deployer) getSwarm(ctx context.Context) (string, error) {
filterArgs := filters.NewArgs()
filterArgs.Add("name", SwarmContainerName)
filterArgs.Add("ancestor", SwarmImageName)
options := types.ContainerListOptions{
Filters: filterArgs,
All: true,
}
containers, err := d.cli.ContainerList(ctx, options)
if err != nil {
return "", err
}
if len(containers) == 0 {
return "", ErrSwarmNotFound
}
return containers[0].ID, nil
}
func (d Deployer) createSwarm(ctx context.Context) (string, error) {
containerConfig := container.Config{
Image: SwarmImageName,
ExposedPorts: nat.PortSet{
"4523/tcp": struct{}{},
"4443/tcp": struct{}{},
"15672/tcp": struct{}{},
},
}
hostConfig := container.HostConfig{
Runtime: "sysbox-runc",
PortBindings: nat.PortMap{
// swarm dockerd
"4523/tcp": []nat.PortBinding{
{
HostIP: "0.0.0.0",
HostPort: "4523",
},
},
// nginx ssl
"4443/tcp": []nat.PortBinding{
{
HostIP: "0.0.0.0",
HostPort: "4443",
},
},
// rabbitmq admin interface
"15672/tcp": []nat.PortBinding{
{
HostIP: "0.0.0.0",
HostPort: "15672",
},
},
},
NetworkMode: "bridge",
}
resp, err := d.cli.ContainerCreate(ctx, &containerConfig, &hostConfig, nil, nil, SwarmContainerName)
if err != nil {
return "", err
}
log.Info().Msg("swarm created successfully")
return resp.ID, nil
}
func (d Deployer) initSwarm(ctx context.Context) error {
req := swarm.InitRequest{
ListenAddr: "0.0.0.0:2377",
AdvertiseAddr: "127.0.0.1",
}
if _, err := d.swarm.cli.SwarmInit(ctx, req); err != nil {
if strings.Contains(err.Error(), "part of a swarm") {
log.Info().Msg("swarm already initialized")
return nil
}
return err
}
return nil
}
func (d Deployer) StopSwarm(ctx context.Context) error {
if d.swarm.ID == "" {
log.Warn().Msg("no swarm registered, can't stop")
return nil
}
options := container.StopOptions{}
if err := d.cli.ContainerStop(ctx, d.swarm.ID, options); err != nil {
return err
}
log.Info().Msg("swarm stopped successfully")
return nil
}
// TODO(rmanach): get a child context instead
func (d *Deployer) init(ctx context.Context) error {
swarmID, err := d.getSwarm(ctx)
if err != nil {
if !errors.Is(err, ErrSwarmNotFound) {
return err
}
}
s := Swarm{swarmID, nil}
if swarmID == "" {
id, err := d.createSwarm(ctx)
if err != nil {
return err
}
s.ID = id
}
if err := d.cli.ContainerStart(ctx, s.ID, types.ContainerStartOptions{}); err != nil {
return err
}
cli, errInit := localcli.GetSwarmClient()
if errInit != nil {
if err := d.StopSwarm(ctx); err != nil {
log.Err(err).Msg("unable to stop the swarm")
}
return errInit
}
s.cli = cli
d.swarm = s
if err := d.initSwarm(ctx); err != nil {
return err
}
log.Info().Msg("deployer successfully initialized")
return nil
}
func (d *Deployer) Deploy(ctx context.Context) error {
if err := d.swarm.deployServices(ctx); err != nil {
return err
}
return nil
}
func (d *Deployer) GetSwarmCLI() (*client.Client, error) {
if d.swarm.cli == nil {
return nil, ErrSwarmClientInit
}
return d.swarm.cli, nil
}

64
deployer/swarm.go Normal file
View File

@ -0,0 +1,64 @@
package deployer
import (
"context"
"localenv/services"
"localenv/utils"
"github.com/docker/docker/client"
"github.com/docker/docker/errdefs"
"github.com/rs/zerolog/log"
)
type Swarm struct {
ID string
cli *client.Client
}
func (s Swarm) deployServices(ctx context.Context) error {
if err := s.createNetworks(ctx); err != nil {
return err
}
if err := s.deployRabbitMQ(ctx); err != nil {
return err
}
if err := s.deployMailhog(ctx); err != nil {
return err
}
if err := s.deployNginx(ctx); err != nil {
return err
}
return nil
}
func (s Swarm) createNetworks(ctx context.Context) error {
log.Info().Str("network", services.NetworkName).Msg("creating network...")
if err := utils.CreateNetwork(ctx, s.cli, services.NetworkName); err != nil {
if !errdefs.IsConflict(err) {
return err
}
}
return nil
}
func (s Swarm) deployRabbitMQ(ctx context.Context) error {
rabbitmq := services.NewRabbitMQ()
return rabbitmq.Deploy(ctx, s.cli)
}
func (s Swarm) deployNginx(ctx context.Context) error {
nginx := services.NewNginx()
return nginx.Deploy(ctx, s.cli)
}
func (s Swarm) deployMailhog(ctx context.Context) error {
mailhog := services.NewMailhog()
return mailhog.Deploy(ctx, s.cli)
}

37
go.mod Normal file
View File

@ -0,0 +1,37 @@
module localenv
go 1.20
require (
github.com/alecthomas/kingpin/v2 v2.3.2
github.com/docker/docker v24.0.2+incompatible
github.com/docker/go-connections v0.4.0
github.com/joho/godotenv v1.5.1
github.com/rs/zerolog v1.29.1
github.com/stretchr/testify v1.8.4
golang.org/x/net v0.6.0
)
require (
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.6.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gotest.tools/v3 v3.4.0 // indirect
)

102
go.sum Normal file
View File

@ -0,0 +1,102 @@
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU=
github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v24.0.2+incompatible h1:eATx+oLz9WdNVkQrr0qjQ8HvRJ4bOOxfzEo8R+dA3cg=
github.com/docker/docker v24.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc=
github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=

77
main.go Normal file
View File

@ -0,0 +1,77 @@
package main
import (
"context"
"os"
"localenv/client"
"localenv/collector"
"localenv/deployer"
"localenv/watchers"
"github.com/alecthomas/kingpin/v2"
"github.com/joho/godotenv"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
var (
watch = kingpin.Flag("watch", "Enable docker local build watcher").Short('w').Bool()
)
func initLogger() {
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
log.Logger = log.With().Caller().Logger().Output(zerolog.ConsoleWriter{Out: os.Stderr})
}
func main() {
initLogger()
ctx := context.Background()
kingpin.Parse()
if err := godotenv.Load(".env"); err != nil {
log.Fatal().AnErr("err", err).Msg("unable to load .env file")
}
hostCLI, err := client.GetHostClient()
if err != nil {
log.Fatal().AnErr("err", err).Msg("unable to connect to the host dockerd")
}
d, err := deployer.NewDeployer(ctx, hostCLI)
if err != nil {
log.Fatal().AnErr("err", err).Msg("unable to initialize the deployer")
}
swarmCLI, err := d.GetSwarmCLI()
if err != nil {
log.Fatal().AnErr("err", err).Msg("unable to get the swarm client")
}
log.Info().Msg("you can now access the swarm ! `make enter`")
images := []string{}
images = append(images, deployer.ImagesDeps...)
c := collector.NewCollector(hostCLI, swarmCLI)
if err := c.DeployImages(ctx, images); err != nil {
log.Fatal().AnErr("err", err).Msg("unable to load/save IO images")
}
if err := d.Deploy(ctx); err != nil {
log.Fatal().AnErr("err", err).Msg("unable to deploy the environment")
}
log.Info().Msg("environment deployed")
watcher, errWatch := watchers.NewWatcher(ctx, hostCLI, swarmCLI)
if errWatch != nil {
log.Fatal().AnErr("err", errWatch).Msg("unable to initialize the watchers")
return
}
if *watch {
watcher.Watch()
}
}

9
nginx/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
FROM nginx:stable-alpine3.17-slim
# embed the configuration directly in the image
COPY nginx/nginx.conf /etc/nginx/nginx.conf
COPY nginx/cors_preflight.conf /etc/nginx/cors_preflight.conf
# init ssl connection
RUN mkdir /etc/nginx/ssl
COPY nginx/ssl/certificate.crt nginx/ssl/private.key /etc/nginx/ssl/

View File

@ -0,0 +1,7 @@
# add_header 'Access-Control-Allow-Origin' 'https://my-front:5173' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, PATCH' always;
add_header 'Access-Control-Allow-Headers' 'Content-Type, Authorization' always;
add_header 'Access-Control-Max-Age' 1728000 always;
# add_header 'Content-Type' 'text/plain charset=UTF-8';
add_header 'Access-Control-Allow-Credentials' 'true' always;
# add_header 'Content-Length' 0;

26
nginx/nginx.conf Normal file
View File

@ -0,0 +1,26 @@
events {
worker_connections 1024;
}
http {
# upstream my-microservice {
# server my-microservice:8083;
# }
include /etc/nginx/cors_preflight.conf;
server {
# location /my-microservice {
# if ($request_method = 'OPTIONS') {
# return 204;
# }
# proxy_pass http://my-microservice/my-microservice;
# }
listen 4443 ssl;
ssl_certificate /etc/nginx/ssl/certificate.crt;
ssl_certificate_key /etc/nginx/ssl/private.key;
resolver 127.0.0.11;
}
}

11
nginx/ssl/generate-certs.bash Executable file
View File

@ -0,0 +1,11 @@
# generate private key
openssl genpkey -algorithm RSA -out private.key -pkeyopt rsa_keygen_bits:2048
# generate cert sign request
openssl req -new -key private.key -out csr.csr -subj "/C=EN/ST=Anywhere/L=Where/O=Organization/OU=localenv/CN=localenv.io"
# generate self signed certificate
openssl x509 -req -days 3650 -in csr.csr -signkey private.key -out certificate.crt
# if you need to import certificate into your browser
# openssl pkcs12 -export -in certificate.crt -inkey private.key -out server.p12

69
services/mailhog.go Normal file
View File

@ -0,0 +1,69 @@
package services
import (
"fmt"
"localenv/utils"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client"
"golang.org/x/net/context"
)
const (
MailhogImageName string = "mailhog/mailhog:latest"
MailhogServiceName string = "mailhog"
)
type Mailhog struct {
Service
}
func NewMailhog() Mailhog {
var nx Mailhog
nx.name = fmt.Sprintf("localenv-%s", MailhogServiceName)
nx.spec = nx.getServiceSpec(
WithIONetwork(),
WithRestartPolicy(),
)
return nx
}
func (n *Mailhog) getServiceSpec(opts ...ServiceOption) swarm.ServiceSpec {
spec := swarm.ServiceSpec{
Annotations: swarm.Annotations{
Name: n.name,
},
TaskTemplate: swarm.TaskSpec{
ContainerSpec: &swarm.ContainerSpec{
Image: MailhogImageName,
Hostname: MailhogServiceName,
},
},
Mode: swarm.ServiceMode{
Replicated: &swarm.ReplicatedService{
Replicas: &SwarmServiceReplicas,
},
},
}
for _, opt := range opts {
opt(&spec)
}
return spec
}
func (n *Mailhog) Deploy(ctx context.Context, cli *client.Client) error {
if err := utils.CreateService(ctx, cli, &n.spec); err != nil {
return err
}
if err := utils.CheckServiceHealthWithRetry(ctx, cli, n.name); err != nil {
return err
}
return nil
}

71
services/nginx.go Normal file
View File

@ -0,0 +1,71 @@
package services
import (
"fmt"
"localenv/utils"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client"
"golang.org/x/net/context"
)
const (
NginxImageName string = "localenv-nginx:latest"
NginxServiceName string = "nginx"
NginxServicePort uint32 = 4443
)
type Nginx struct {
Service
}
func NewNginx() Nginx {
var nx Nginx
nx.name = fmt.Sprintf("localenv-%s", NginxServiceName)
nx.spec = nx.getServiceSpec(
WithIONetwork(),
WithHostEndpoint(NginxServicePort),
WithRestartPolicy(),
)
return nx
}
func (n *Nginx) getServiceSpec(opts ...ServiceOption) swarm.ServiceSpec {
spec := swarm.ServiceSpec{
Annotations: swarm.Annotations{
Name: n.name,
},
TaskTemplate: swarm.TaskSpec{
ContainerSpec: &swarm.ContainerSpec{
Image: NginxImageName,
},
},
Mode: swarm.ServiceMode{
Replicated: &swarm.ReplicatedService{
Replicas: &SwarmServiceReplicas,
},
},
}
for _, opt := range opts {
opt(&spec)
}
return spec
}
func (n *Nginx) Deploy(ctx context.Context, cli *client.Client) error {
if err := utils.CreateService(ctx, cli, &n.spec); err != nil {
return err
}
if err := utils.CheckServiceHealthWithRetry(ctx, cli, n.name); err != nil {
return err
}
return nil
}

90
services/postgres.go Normal file
View File

@ -0,0 +1,90 @@
package services
import (
"fmt"
"localenv/utils"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client"
"golang.org/x/net/context"
)
const (
PostgresImageName string = "postgres:13.4-alpine"
PostgresServicePort uint32 = 5432
)
type Postgres struct {
Service
}
func NewPostgres(serviceName string) Postgres {
var pg Postgres
pg.name = fmt.Sprintf("localenv-postgres-%s", serviceName)
pg.spec = pg.getServiceSpec(
serviceName,
WithNetwork(serviceName),
WithRestartPolicy(),
)
return pg
}
func (p *Postgres) getServiceSpec(name string, opts ...ServiceOption) swarm.ServiceSpec {
spec := swarm.ServiceSpec{
Annotations: swarm.Annotations{
Name: p.name,
},
TaskTemplate: swarm.TaskSpec{
ContainerSpec: &swarm.ContainerSpec{
Image: PostgresImageName,
Hostname: fmt.Sprintf("pg-%s", name),
Env: []string{
"POSTGRES_HOSTNAME=localhost",
fmt.Sprintf("POSTGRES_PORT=%d", PostgresServicePort),
fmt.Sprintf("POSTGRES_DB=%s", name),
"POSTGRES_USER=test",
"POSTGRES_PASSWORD=test",
},
Mounts: []mount.Mount{
{
Type: mount.TypeVolume,
Source: p.name,
Target: "/var/lib/postgresql/data/",
},
},
},
},
Mode: swarm.ServiceMode{
Replicated: &swarm.ReplicatedService{
Replicas: &SwarmServiceReplicas,
},
},
}
for _, opt := range opts {
opt(&spec)
}
return spec
}
func (p *Postgres) Deploy(ctx context.Context, cli *client.Client) error {
if err := utils.CreateVolume(ctx, cli, p.name); err != nil {
return err
}
if err := utils.CreateService(ctx, cli, &p.spec); err != nil {
return err
}
if err := utils.CheckServiceHealthWithRetry(ctx, cli, p.name); err != nil {
return err
}
return nil
}

85
services/rabbitmq.go Normal file
View File

@ -0,0 +1,85 @@
package services
import (
"fmt"
"localenv/utils"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client"
"golang.org/x/net/context"
)
const (
RabbitMQImageName string = "rabbitmq:3-management-alpine"
RabbitMQServiceName string = "rabbitmq"
RabbitMQServicePort uint32 = 5672
RabbitMQServiceManagementPort uint32 = 15672
)
type RabbitMQ struct {
Service
}
func NewRabbitMQ() RabbitMQ {
var rq RabbitMQ
rq.name = fmt.Sprintf("localenv-%s", RabbitMQServiceName)
rq.spec = rq.getServiceSpec(
WithIONetwork(),
WithHostEndpoint(RabbitMQServiceManagementPort),
WithRestartPolicy(),
)
return rq
}
func (r *RabbitMQ) getServiceSpec(opts ...ServiceOption) swarm.ServiceSpec {
spec := swarm.ServiceSpec{
Annotations: swarm.Annotations{
Name: r.name,
},
TaskTemplate: swarm.TaskSpec{
ContainerSpec: &swarm.ContainerSpec{
Image: RabbitMQImageName,
Hostname: RabbitMQServiceName,
Env: []string{
"RABBITMQ_DEFAULT_USER=intercloud",
"RABBITMQ_DEFAULT_PASS=intercloud",
},
Mounts: []mount.Mount{
{
Type: mount.TypeVolume,
Source: "mqdata",
Target: "/var/lib/rabbitmq",
},
},
},
},
Mode: swarm.ServiceMode{
Replicated: &swarm.ReplicatedService{
Replicas: &SwarmServiceReplicas,
},
},
}
for _, opt := range opts {
opt(&spec)
}
return spec
}
func (r *RabbitMQ) Deploy(ctx context.Context, cli *client.Client) error {
if err := utils.CreateService(ctx, cli, &r.spec); err != nil {
return err
}
if err := utils.CheckServiceHealthWithRetry(ctx, cli, r.name); err != nil {
return err
}
return nil
}

157
services/service.go Normal file
View File

@ -0,0 +1,157 @@
package services
import (
"context"
"errors"
"fmt"
"time"
"localenv/utils"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client"
)
const NetworkName = "localenv"
var (
RetryServiceAttempts uint64 = 0
SwarmServiceReplicas uint64 = 1
RetryServiceDelay = 20 * time.Second
ErrNoRunningContainer = errors.New("no running container")
)
type ServiceOption func(spec *swarm.ServiceSpec)
func WithIONetwork() ServiceOption {
return WithNetwork(NetworkName)
}
func WithNetwork(name string) ServiceOption {
return func(spec *swarm.ServiceSpec) {
spec.Networks = append(spec.Networks,
swarm.NetworkAttachmentConfig{
Target: name,
Aliases: []string{name},
},
)
}
}
func WithHostEndpoint(port uint32) ServiceOption {
return func(spec *swarm.ServiceSpec) {
spec.EndpointSpec = &swarm.EndpointSpec{
Mode: "vip",
Ports: []swarm.PortConfig{
{
Protocol: "tcp",
TargetPort: port,
PublishedPort: port,
PublishMode: "host",
},
},
}
}
}
func WithRestartPolicy() ServiceOption {
return func(spec *swarm.ServiceSpec) {
spec.TaskTemplate.RestartPolicy = &swarm.RestartPolicy{
Condition: swarm.RestartPolicyConditionOnFailure,
MaxAttempts: &RetryServiceAttempts,
Delay: &RetryServiceDelay,
}
}
}
func WithPostgres(name string) ServiceOption {
return func(spec *swarm.ServiceSpec) {
if spec.TaskTemplate.ContainerSpec == nil {
return
}
spec.TaskTemplate.ContainerSpec.Env = append(
spec.TaskTemplate.ContainerSpec.Env,
"DB_TYPE=postgres",
fmt.Sprintf("POSTGRES_HOSTNAME=pg-%s", name),
fmt.Sprintf("POSTGRES_PORT=%d", PostgresServicePort),
fmt.Sprintf("POSTGRES_DB=%s", name),
"POSTGRES_USER=test",
"POSTGRES_PASSWORD=test",
)
}
}
func WithRabbitMQ() ServiceOption {
return func(spec *swarm.ServiceSpec) {
if spec.TaskTemplate.ContainerSpec == nil {
return
}
spec.TaskTemplate.ContainerSpec.Env = append(
spec.TaskTemplate.ContainerSpec.Env,
fmt.Sprintf("RABBITMQ_ENDPOINT=amqp://%s:%d", RabbitMQServiceName, RabbitMQServicePort),
"RABBITMQ_USERNAME=intercloud",
"RABBITMQ_PASSWORD=intercloud",
)
}
}
type Servicer interface {
Deploy(ctx context.Context, cli *client.Client) error
}
type Service struct {
spec swarm.ServiceSpec
name string
}
func (p *Service) GetBaseServiceSpec(serviceName, hostname, imageName, port string, command []string) swarm.ServiceSpec {
spec := swarm.ServiceSpec{
Annotations: swarm.Annotations{
Name: serviceName,
},
TaskTemplate: swarm.TaskSpec{
ContainerSpec: &swarm.ContainerSpec{
Hostname: hostname,
Image: imageName,
Env: []string{
fmt.Sprintf("PORT=%s", port),
},
Command: command,
},
},
Mode: swarm.ServiceMode{
Replicated: &swarm.ReplicatedService{
Replicas: &SwarmServiceReplicas,
},
},
}
return spec
}
func Deploy(ctx context.Context, cli *client.Client, spec *swarm.ServiceSpec, dependencies []Servicer) error {
err := utils.CheckServiceHealth(ctx, cli, spec.Annotations.Name)
if err == nil {
return nil
}
if !errors.Is(err, utils.ErrServiceNotFound) {
return err
}
for _, deps := range dependencies {
if err := deps.Deploy(ctx, cli); err != nil {
return err
}
}
if err := utils.CreateService(ctx, cli, spec); err != nil {
return err
}
return nil
}

4
swarm/Dockerfile Normal file
View File

@ -0,0 +1,4 @@
FROM nestybox/ubuntu-jammy-systemd-docker
# rewrite the systemd service to allow tcp connection
RUN sed -i 's/dockerd -H/dockerd --tls=false -H tcp:\/\/0.0.0.0:4523 -H/g' /lib/systemd/system/docker.service

237
utils/utils.go Normal file
View File

@ -0,0 +1,237 @@
package utils
import (
"context"
"errors"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/api/types/volume"
"github.com/docker/docker/client"
"github.com/docker/docker/errdefs"
"github.com/rs/zerolog/log"
)
var (
ErrServiceNotFound = errors.New("unable to found the service")
ErrServiceCommandFailed = errors.New("command failed on the container")
ErrServiceUnhealthy = errors.New("service unhealthy")
)
type FnRetry func(ctx context.Context) error
func Retry(ctx context.Context, fnRetry FnRetry, waitDuration time.Duration, maxRetry int) (err error) {
for i := 0; i < maxRetry; i++ {
if err = fnRetry(ctx); err != nil {
time.Sleep(waitDuration)
continue
}
return nil
}
return err
}
func CreateNetwork(ctx context.Context, cli *client.Client, networkName string) error {
networkSpec := types.NetworkCreate{
Driver: "overlay",
}
_, err := cli.NetworkCreate(ctx, networkName, networkSpec)
if err != nil {
return err
}
return nil
}
func FilterImagesByName(ctx context.Context, cli *client.Client, name string) ([]types.ImageSummary, error) {
images, err := cli.ImageList(ctx, types.ImageListOptions{All: true})
if err != nil {
return nil, err
}
var filteredImages []types.ImageSummary
for i := range images {
for _, tag := range images[i].RepoTags {
if tag == name {
filteredImages = append(filteredImages, images[i])
break
}
}
}
return filteredImages, nil
}
func CreateService(ctx context.Context, cli *client.Client, spec *swarm.ServiceSpec) error {
serviceName := spec.Annotations.Name
log.Info().Str("service", serviceName).Msg("creating service...")
response, err := cli.ServiceCreate(ctx, *spec, types.ServiceCreateOptions{})
if err != nil {
if !errdefs.IsConflict(err) {
return err
}
log.Info().Str("service", serviceName).Msg("service already deployed")
return nil
}
log.Info().Str("service", serviceName).Str("id", response.ID).Msg("service deployed")
return nil
}
func GetServiceByName(ctx context.Context, cli *client.Client, name string) (swarm.Service, error) {
servicesFilters := filters.NewArgs()
servicesFilters.Add("name", name)
options := types.ServiceListOptions{
Filters: servicesFilters,
Status: true,
}
services, err := cli.ServiceList(ctx, options)
if err != nil {
return swarm.Service{}, err
}
if len(services) != 1 {
return swarm.Service{}, ErrServiceNotFound
}
return services[0], nil
}
func CheckServiceHealth(ctx context.Context, cli *client.Client, serviceName string) error {
log.Info().Str("service", serviceName).Msg("healthchecking...")
service, err := GetServiceByName(ctx, cli, serviceName)
if err != nil {
return err
}
if service.ServiceStatus != nil && service.ServiceStatus.RunningTasks != 0 {
log.Info().Str("service", serviceName).Msg("service is running")
return nil
}
log.Info().Str("service", serviceName).Msg("service not already started, retrying...")
return errors.New("service unhealthy")
}
func CheckServiceHealthWithRetry(ctx context.Context, cli *client.Client, serviceName string) error {
fnRetry := func(ctx context.Context) error {
return CheckServiceHealth(ctx, cli, serviceName)
}
waitDuration := 5 * time.Second
maxRetry := 15
return Retry(ctx, fnRetry, waitDuration, maxRetry)
}
func CheckAndDeleteCompletedService(ctx context.Context, cli *client.Client, name string) error {
service, err := GetServiceByName(ctx, cli, name)
if err != nil {
return err
}
fnRetry := func(ctx context.Context) error {
taskFilters := filters.NewArgs()
taskFilters.Add("service", service.ID)
tasks, err := cli.TaskList(ctx, types.TaskListOptions{
Filters: taskFilters,
})
if err != nil {
return err
}
completeTasks := 0
for idx := range tasks {
if tasks[idx].Status.State == swarm.TaskStateComplete {
completeTasks++
}
}
if completeTasks == 0 {
return errors.New("no completed tasks")
}
return nil
}
maxAttempts := 20
if err := Retry(ctx, fnRetry, time.Second, maxAttempts); err != nil {
return err
}
if err := cli.ServiceRemove(ctx, service.ID); err != nil {
return err
}
return nil
}
// UpdateServiceByName updates force a service by its name. It will remove and recreate the service.
func UpdateServiceByName(ctx context.Context, cli *client.Client, serviceName string) error {
srv, err := GetServiceByName(ctx, cli, serviceName)
if err != nil {
return err
}
if err := RemoveService(ctx, cli, srv.ID); err != nil {
return err
}
if err := CreateService(ctx, cli, &srv.Spec); err != nil {
return err
}
return nil
}
func RemoveService(ctx context.Context, cli *client.Client, serviceID string) error {
fnRetry := func(ctx context.Context) error {
if err := cli.ServiceRemove(ctx, serviceID); err != nil {
return err
}
return nil
}
maxRetry := 10
if err := Retry(ctx, fnRetry, time.Second, maxRetry); err != nil {
return err
}
return nil
}
func RemoveImage(ctx context.Context, cli *client.Client, imageID string) error {
fnRetry := func(ctx context.Context) error {
if _, err := cli.ImageRemove(ctx, imageID, types.ImageRemoveOptions{}); err != nil {
if errdefs.IsConflict(err) {
log.Warn().Str("image", imageID).Msg("image is using")
}
return err
}
return nil
}
maxRetry := 10
if err := Retry(ctx, fnRetry, time.Second, maxRetry); err != nil {
return err
}
return nil
}
func CreateVolume(ctx context.Context, cli *client.Client, volumeName string) error {
if _, err := cli.VolumeCreate(ctx, volume.CreateOptions{Name: volumeName}); err != nil {
if !errdefs.IsConflict(err) {
return err
}
}
return nil
}

186
watchers/watcher.go Normal file
View File

@ -0,0 +1,186 @@
package watchers
import (
"context"
"errors"
"fmt"
"strings"
"localenv/collector"
"localenv/services"
"localenv/utils"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/events"
"github.com/docker/docker/client"
"github.com/rs/zerolog/log"
)
var (
ErrUninitializeEvents = errors.New("uninitialize events channel")
ErrClientConnection = errors.New("unable to connect to the docker daemon")
ErrParseServiceName = errors.New("unable to parse service name")
ErrServiceEmptyImage = errors.New("no image found in service")
)
func parseImageName(imageName string) string {
if imageName == "" {
return ""
}
values := strings.Split(imageName, "/")
r := values[len(values)-1:][0]
r = strings.TrimPrefix(r, "localenv-")
r = strings.TrimSuffix(r, ":latest")
r = strings.TrimSuffix(r, "-local")
return fmt.Sprintf("localenv-%s", r)
}
type Watcher struct {
ctx context.Context
fnCancel context.CancelCauseFunc
hostCLI *client.Client
swarmCLI *client.Client
collector collector.Collector
chHostEvents <-chan events.Message
chHostEventsErr <-chan error
chSwarmEvents <-chan events.Message
chSwarmEventsErr <-chan error
chDone chan struct{}
}
func NewWatcher(ctx context.Context, hostCLI, swarmCLI *client.Client) (Watcher, error) {
chidlCtx, cancel := context.WithCancelCause(ctx)
chHostEvents, chHostEventsErr := hostCLI.Events(ctx, types.EventsOptions{})
chSwarmEvents, chSwarmEventsErr := swarmCLI.Events(ctx, types.EventsOptions{})
c := collector.NewCollector(hostCLI, swarmCLI)
return Watcher{
ctx: chidlCtx,
fnCancel: cancel,
hostCLI: hostCLI,
swarmCLI: swarmCLI,
collector: c,
chHostEvents: chHostEvents,
chHostEventsErr: chHostEventsErr,
chSwarmEvents: chSwarmEvents,
chSwarmEventsErr: chSwarmEventsErr,
chDone: make(chan struct{}),
}, nil
}
// TODO(rmanach): in order to watch image built, force the image tag:
// @docker image tag my-image:latest my-image:latest
func (w *Watcher) watchHostEvents() error {
if w.chHostEvents == nil {
return fmt.Errorf("%w: host events", ErrUninitializeEvents)
}
if w.chHostEventsErr == nil {
return fmt.Errorf("%w: host events err", ErrUninitializeEvents)
}
go func() {
for {
select {
case evt := <-w.chHostEvents:
log.Info().Str("event", evt.Actor.ID).Str("action", evt.Action).Str("type", evt.Type).Msg("host event received")
switch evt.Type {
case events.ImageEventType:
// handle only this action, inf loop with the collector
if evt.Action == "tag" {
if err := w.handleImageEvent(w.ctx, evt.Actor.ID); err != nil {
log.Err(err).Str("image", evt.Actor.ID).Msg("unable to handle image")
}
}
default:
}
case err := <-w.chHostEventsErr:
log.Err(err).Msg("error occurred in host stream events")
case <-w.ctx.Done():
log.Error().Str("events", "host").Msg("context done")
return
}
}
}()
return nil
}
// handleImageEvent handles image tag events.
// It will build the service name from the tag, delete the service and the image on the swarm
// and thanks to the `collector` redeploys the image and restart the service.
//
// TODO(rmanach): brute update, should be nice to compare image sha before update.
// TODO(rmanach): deploy with migrations and data
func (w *Watcher) handleImageEvent(ctx context.Context, imageID string) error {
inspect, _, err := w.hostCLI.ImageInspectWithRaw(ctx, imageID)
if err != nil {
return err
}
imageNameWithTag := inspect.RepoTags[0]
serviceName := parseImageName(imageNameWithTag)
srv, err := utils.GetServiceByName(ctx, w.swarmCLI, serviceName)
if err != nil {
return err
}
if err := w.swarmCLI.ServiceRemove(ctx, srv.ID); err != nil {
return err
}
if srv.Spec.TaskTemplate.ContainerSpec == nil {
return ErrServiceEmptyImage
}
if err := utils.RemoveImage(ctx, w.swarmCLI, srv.Spec.TaskTemplate.ContainerSpec.Image); err != nil {
return err
}
if err := w.collector.DeployImages(ctx, []string{imageNameWithTag}); err != nil {
return err
}
if err := utils.CreateService(ctx, w.swarmCLI, &srv.Spec); err != nil {
return err
}
if err := utils.CheckServiceHealthWithRetry(ctx, w.swarmCLI, srv.Spec.Name); err != nil {
return err
}
log.Info().Str("service", serviceName).Str("image", imageNameWithTag).Msg("service update")
// nginx needs update too to avoid 502 error
if err := utils.UpdateServiceByName(ctx, w.swarmCLI, fmt.Sprintf("localenv-%s", services.NginxServiceName)); err != nil {
return err
}
return nil
}
// TODO(rmanach): impl a graceful stop
func (w *Watcher) Stop() {
w.fnCancel(nil)
}
func (w *Watcher) Watch() {
if err := w.watchHostEvents(); err != nil {
log.Err(err).Msg("while watching host events")
w.fnCancel(err)
return
}
log.Info().Msg("watcher is listening events...")
<-w.ctx.Done()
}

53
watchers/watcher_test.go Normal file
View File

@ -0,0 +1,53 @@
package watchers
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestParseImageName(t *testing.T) {
testCases := []struct {
imageName string
expect string
}{
{
imageName: "",
expect: "",
},
{
imageName: "toto",
expect: "localenv-toto",
},
{
imageName: "company/my-microservice:latest",
expect: "localenv-my-microservice",
},
{
imageName: "localenv-nginx",
expect: "localenv-nginx",
},
{
imageName: "localenv-nginx:latest",
expect: "localenv-nginx",
},
{
imageName: "mailhog/mailhog:latest",
expect: "localenv-mailhog",
},
{
imageName: "company/my-super-ms",
expect: "localenv-my-super-ms",
},
{
imageName: "registry.mycompany.com/company/test/build-images/microservice-base",
expect: "localenv-microservice-base",
},
}
for _, tc := range testCases {
imageName := parseImageName(tc.imageName)
assert.Equal(t, tc.expect, imageName)
}
}