dps: Support für authelia_policy=none

_get_container.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+if [ -z "$1" ]; then
+	base=`realpath . --relative-to="/data/containers" | cut -d"/" -f1`
+	if [ "$base" = ".." -o "$base" = "." ]; then
+		echo "No container given and not within /data/containers"
+		exit 1
+	fi
+	container="$base"
+	echo "Working on container $container."
+else
+	container="$1"
+fi

_install.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+SOURCE=$(pwd)
+TARGET="/usr/local/bin"
+
+if [ ! -e "$SOURCE/dbash" ]; then
+	echo "Please cd to the directory containing this file and try to run _install.sh again."
+	exit 1
+fi
+
+ln -sf "$SOURCE/dbash"      "$TARGET/dbash"
+ln -sf "$SOURCE/dhelp"      "$TARGET/dhelp"
+ln -sf "$SOURCE/dlogs"      "$TARGET/dlogs"
+ln -sf "$SOURCE/dlogsl"     "$TARGET/dlogsl"
+ln -sf "$SOURCE/drl"        "$TARGET/drl"
+ln -sf "$SOURCE/dps.rb"     "$TARGET/dps"
+ln -sf "$SOURCE/dsh"        "$TARGET/dsh"
+ln -sf "$SOURCE/dstats"     "$TARGET/dstats"
+ln -sf "$SOURCE/dup.rb"     "$TARGET/dup"
+ln -sf "$SOURCE/ofelias.rb" "$TARGET/ofelias"

dbash

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+. $(dirname "$(readlink -f "$0")")/_get_container.sh
+
+docker exec -it "$container" /bin/bash

dhelp

@@ -0,0 +1,9 @@
+echo "Docker-Tools"
+echo
+echo "dps        - Shows a tabular list of running containers."
+echo "dlogs      - Show latest log entries of container."
+echo "dlogsl     - Shows latest log entries using less."
+echo "dup        - Creates and runs containers from YAML files."
+echo "dbash      - Starts a bash shell in the container."
+echo "dsh        - Starts a sh shell in the container."
+echo "dstats     - Shows stats for all running containers."

dlogs

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+. $(dirname "$(readlink -f "$0")")/_get_container.sh
+
+docker logs --tail=100 -f "$container"

dlogsl

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+. $(dirname "$(readlink -f "$0")")/_get_container.sh
+
+docker logs -f "$container" 2>&1 | less

dps.rb

@@ -0,0 +1,144 @@
+#!/usr/bin/env ruby
+require 'json'
+require 'socket'
+require 'net/http'
+
+COLOR_GREEN = ""
+COLOR_RED   = ""
+COLOR_GRAY  = ""
+COLOR_RESET = ""
+
+class Container
+	FIELDS = [:state, :dup, :ouroboros, :name, :vhosts, :authelia, :status, :image, :ports]
+	@@lengths = Hash.new(0)
+	
+	def initialize(data)
+		@strings = {}
+		@data = data
+		
+		set :state, (@data[:State]=="running" ? COLOR_GREEN : COLOR_RED) + @data[:State].capitalize + COLOR_RESET
+		set :dup, @data[:Labels][:"de.fabianonline.dup"]=="true" ? "✓" : ""
+		set :ouroboros, @data[:Labels][:"com.centurylinklabs.watchtower.enable"]=="true" ? "✓" : ""
+		set :name, @data[:Names].first[1..-1]
+		
+		vh = @data[:Labels][:"nginx_virtual_host"]&.split(",")&.map{|h| h.split("@").first.strip}
+		if vh.nil? || vh.count==0
+			vh = ""
+		elsif vh.count==1
+			vh = vh.first
+		else
+			if FULL
+				vh = vh.join(", ")
+			else
+				vh = vh.first + ",…"
+			end
+		end
+		
+		set :vhosts, vh
+		
+		p = @data[:Ports].sort_by{|p| p[:PrivatePort]}.collect do |p|
+			type = p[:Type]=="tcp" ? "" : "/#{p[:Type]}"
+			if p[:PrivatePort]==p[:PublicPort]
+				"#{COLOR_GREEN}#{p[:PrivatePort]}#{type}"
+			elsif p[:PublicPort]==nil
+				"#{COLOR_GRAY}#{p[:PrivatePort]}#{type}"
+			else
+				"#{COLOR_RED}#{p[:PrivatePort]}->#{p[:PublicPort]}#{type}"
+			end
+		end.uniq.join(" ")
+		
+		#set :nginx_allow, @data[:Labels][:"nginx_allow"]
+		if vh==""
+			set :authelia, ""
+		elsif @data[:Labels][:"authelia_policy"]=="one_factor"
+			set :authelia, "1#{COLOR_GRAY}FA#{COLOR_RESET}"
+		elsif @data[:Labels][:"authelia_policy"]=="two_factor"
+			set :authelia, "2#{COLOR_GRAY}FA#{COLOR_RESET}"
+		elsif @data[:Labels][:"authelia_policy"]=="bypass"
+			set :authelia, "#{COLOR_GREEN}BYP#{COLOR_RESET}"
+		elsif @data[:Labels][:"authelia_policy"]=="none"
+			set :authelia, "#{COLOR_GREEN}NONE#{COLOR_RESET}"
+		else
+			set :authelia, "#{COLOR_RED}?#{COLOR_RESET}"
+		end
+
+		st = @data[:Status]
+		if st.end_with?("(healthy)")
+			st = COLOR_GREEN + st.gsub("(healthy)", "(v)") + COLOR_RESET
+		elsif st.end_with?("(unhealthy)")
+			st = COLOR_RED + st.gsub("(unhealthy)", "(x)") + COLOR_RESET
+		end
+		set :status, st
+		im = @data[:Image]
+		im = im.split(":")[0] unless FULL
+		set :image, im
+		set :ports, p
+	end
+	
+	def set(key, value)
+		@data[key] = value
+		@@lengths[key] = [@@lengths[key], value.cleaned.length].max if value
+	end
+	
+	def self.lengths; FIELDS.map{|k| @@lengths[k]}; end
+	
+	def data
+		FIELDS.map{|k| @data[k]}
+	end
+end
+
+class String
+	def cleaned
+		self.gsub(/\[.+?m/, "")
+	end
+
+	def fix_length(len)
+		l = self.cleaned.length
+		shortened = self.length - l
+		
+		if l >= len
+			return self.slice(0, len + shortened)
+		else
+			return self + (" " * (len - l))
+		end
+	end
+end
+
+def fetch_data
+	socket = UNIXSocket.new("/var/run/docker.sock")
+	request = "GET /v1.24/containers/json?all=1 HTTP/1.0\r\n\r\n"#Host: localhost\r\n\r\n"
+	socket.write(request)
+
+	response = ""
+
+	loop do
+		break if socket.eof?
+		line = socket.gets
+		break if line=="\r\n"
+	end
+
+	until socket.eof?
+		line = socket.gets
+		response += line
+	end
+
+	return JSON.parse(response, symbolize_names: true)
+end
+
+FULL = ARGV[0]=="--full"
+
+data = fetch_data
+data = data.map{|c| Container.new(c).data}
+
+lengths = Container.lengths
+
+data.sort_by{|d| d[3]}.each do |row|
+	row.each_with_index do |d, index|
+		if index==row.length-1
+			print d.to_s
+		else
+			print d.to_s.fix_length(lengths[index]) + "  "
+		end
+	end
+	puts
+end

drl

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+. $(dirname "$(readlink -f "$0")")/_get_container.sh
+
+docker restart "$container" && docker logs --tail=100 -f "$container"

dsh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+. $(dirname "$(readlink -f "$0")")/_get_container.sh
+
+docker exec -it "$container" /bin/sh

dstats

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker stats $(docker inspect --format "{{.Name}}" $(docker ps -q))

dup.rb

@@ -7,6 +7,16 @@ require 'ostruct'
 require 'json'
 
 MAPPINGS = [
+	# Syntax:
+	# [name_in_yml, docker_option_name, lambda_for_creating_yml_for_container=nil, options={}]
+	# Options are:
+	#   type: Sets type of this option
+	#     switch: Only adds the docker_option_name to the command, ignoring the value.
+	#     name: Value will be set to the name of the container (from yml filename or root element)
+	#     hidden: Will not be added to the run command
+	#   allow_empty: May be empty.
+	#   escape: If set to false, will not be shellescaped
+
 	['stdin_open', '--interactive', lambda{|c,i| c.Config.OpenStdin}, {:type=>:switch}],
 	['tty', '--tty', lambda{|c,i| c.Config.Tty}, {:type=>:switch}],
 	#['detach', '--detach', {:type=>:switch}],
@@ -23,24 +33,29 @@ MAPPINGS = [
 	['devices', '--device'],
 	['net', '--net'],
 	['ipv6', '--ipv6'],
+	['user', '--user'],
 	['networks', '--network', lambda{|c,i| c.NetworkSettings.Networks.to_h.keys.map(&:to_s)}, {:type=>:hidden}],
 	['entrypoint', '--entrypoint', lambda{|c,i| (ep=c.Config.Entrypoint) == i.Config.Entrypoint ? nil : ep}, {:allow_empty=>true}],
 	['labels', '--label', lambda{|c,i| l=Hash[c.Config.Labels.to_h.delete_if{|k,v| i.Config.Labels[k]==v rescue false}.map{|k,v| [k.to_s, v]}] ; l}],
 	['hostname', '--hostname'],
+	['shm_size', '--shm-size'],
 	['working_dir', '--workdir', lambda{|c,i| (wd=c.Config.WorkingDir) == i.Config.WorkingDir ? nil : wd}],
 	['init', '--init', lambda{|c,i| c.Config.Init}, {:type=>:switch}],
 	['image', nil, lambda{|c,i| c.Config.Image}],
-	['command', nil, lambda{|c,i| (cmd=c.Config.Cmd.join(" ") rescue nil) == (i.Config.Cmd.join(" ") rescue nil)? nil : cmd}, {:escape=>false}]
+	['command', nil, lambda{|c,i| (cmd=c.Config.Cmd.join(" ") rescue nil) == (i.Config.Cmd.join(" ") rescue nil)? nil : cmd}, {:escape=>false}],
+	['test', nil, nil, {type: :hidden}],
+	['auto_update', nil, nil, {type: :hidden}],
+	['ofelia', nil, nil, {type: :hidden}]
 ]
 
 
 def get_sample(name="container")
 	return <<~HEREDOC
-		#{name}:
 		image: "my/image:1.2.3"
 		restart: always
 		detach: true
 		test: true # Having this set to true will prevent dup from creating a label de.fabianonline.dup
+		auto_update: true # Enabling this adds a label to allow watchtower to automatically update this container
 		  
 		ports:
 		  - "1234:1234"
@@ -54,24 +69,34 @@ def get_sample(name="container")
 		volumes:
 		  - "/etc/localtime:/etc/localtime:ro"
 		  
-		  links:
-		    - "another_container"
-		
 		labels:
-		    com.centurylinklabs.watchtower.enable: true
-		    nginx_virtual_host: container.home.fabianonline.de
+		  # Syntax:
+		  #   <hostname>: Uses the given hostname and the default cert (*.schle.nz)
+		  #   <hostname>@: Uses the given hostname and the cert with the same name
+		  #   <hostname>@<cert>: Uses the given hostname and cert
+		  nginx_virtual_host: container.home.schle.nz
 		  nginx_port: 80
-		    nginx_allow: fabian # "user" or "@group" or "user, user, @group" or "all"
+		  nginx_additional_ports: "443, 5050"
 		  nginx_no_robots: true
 		  nginx_public_paths: "/public, /api"
-		    nginx_type: # "http" (default), "fastcgi", "skip" (doesn't create any entries)
+		  nginx_type: # "http" (default), "https", "fastcgi", "skip" (doesn't create any entries)
+		  nginx_client_max_body_size: "25M"
+		  # Syntax:
+		  #   user:<username>
+		  #   group:<groupname>
+		  authelia_allow: "user:fabian"
+		  # Policies:
+		  #   one_factor
+		  #   two_factor
+		  #   bypass
+		  authelia_policy: one_factor
 		  
 		networks:
 		  - "nginx"
 		  - "mosquitto"
 		  - "bridge"
 		  
-		  test: true
+		user: "1000:1000"
 		command: "/bin/bash"
 		entrypoint: "/script.sh"
 		build: "/data/dir"
@@ -85,6 +110,7 @@ def get_sample(name="container")
 		remove: false
 		stop_signal: SIGUSR1
 		stop_grace_period: 5
+		shm_size: 256M
 		
 		before_build:
 		  - "echo 'Starting build'"
@@ -98,6 +124,15 @@ def get_sample(name="container")
 		  
 		devices:
 		  - "/dev/ttyUSB0:/dev/ttyUSB0"
+		
+		ofelia:
+		  - name: update # No spaces!
+		    schedule: "@every 4h" # go cron format! See https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc
+		    command: uname -a
+		    user: foo
+		    tty: false
+		    no-overlap: true
+		  - name: Test 2
 		HEREDOC
 end
 
@@ -115,18 +150,31 @@ def action_help
 	puts "-l, --list              Lists all available containers."
 	puts "    --host              Starts a container with net: host - which implies not using ports, networks and/or links."
 	puts "    --regenerate        Generates YAML content from a running container."
+	puts "    --all               Use all containers defined in yml files."
+	puts "-v, --verbose           More verbose output."
+	puts "-q, --quiet             Don't open log after starting a new container."
 	exit 1
 end
 
-def run_cmd(cmd, ignore_returnvalue=false, catch_interrupt=false)
-	puts "+ #{cmd}" if $dry_run
+def run_cmd(cmd, ignore_returnvalue=false, catch_interrupt=false, ignore_dry_run: false)
+	verbose "+ #{cmd}"
 	returnvalue=false
 	begin
-		returnvalue = $dry_run ? true : system("bash -c #{cmd.shellescape}")
+		returnvalue = ($dry_run && !ignore_dry_run) ? true : system("bash -c #{cmd.shellescape}")
 	rescue Interrupt
 		raise if not catch_interrupt
 	end
 	raise "Command returned a non-zero exit value." if returnvalue!=true && !ignore_returnvalue
+	return returnvalue
+end
+
+def collect_commands(&block)
+	$commands = []
+	def run_cmd(cmd, ignore_returnvalue=false, catch_interrupt=false, ignore_dry_run: false); $commands<<cmd; return 0; end
+
+	yield
+
+	return $commands
 end
 
 def esc(obj, escape=true)
@@ -192,8 +240,7 @@ end
 def action_regenerate(container)
 	container.each do |c|
 		data = c.regenerate
-		puts data.to_yaml
-		puts
+		p collect_commands{ c.run }
 	end
 end
 
@@ -202,6 +249,10 @@ class Container
 		@name = name
 	end
 	
+	def exists?
+		run_cmd("docker inspect --format='1' #{@name.shellescape} >/dev/null 2>&1", true, ignore_dry_run: true)
+	end
+	
 	def load
 		raise "File #{self.filename} not found." unless File.exists?(self.filename)
 
@@ -217,23 +268,51 @@ class Container
 	
 	def filename; "%s/%s.yml" % [$base_dir, @name]; end
 
-	def build_run_command
-		cmd = ["docker", "create"]
-		cmd << "--net" << @data["networks"][0] if @data["networks"] && !$net_host
-		cmd << "--net" << "host" if $net_host
-		
-		if !@data["test"]
-			if @data["labels"]==nil
-				@data["labels"] = ["de.fabianonline.dup=true"]
-			elsif @data["labels"].is_a? Array
-				@data["labels"] << "de.fabianonline.dup=true"
+	def add_label(key, value)
+		if @data["labels"].is_a? Array
+			@data["labels"] << "#{key}=#{value}"
 		elsif @data["labels"].is_a? Hash
-				@data["labels"]["de.fabianonline.dup"] = "true"
+			@data["labels"][key] = value
 		else
 			raise "data['labels'] is of an unexpected type: #{@data["labels"].class}"
 		end
 	end
 	
+	def build_run_command
+		$net_host = true if @data["net"]=="host"
+		cmd = ["docker", "create"]
+		cmd << "--net" << @data["networks"][0] if @data["networks"] && @data["networks"].count>0 && !$net_host
+		cmd << "--net" << "host" if $net_host
+
+		@data["labels"] ||= []
+		
+		if !@data["test"]
+			add_label("de.fabianonline.dup", "true")
+		end
+
+		if @data["auto_update"]
+			add_label("com.centurylinklabs.watchtower.enable", "true")
+		end
+
+		if @data["ofelia"] && @data["ofelia"].count > 0
+			add_label("ofelia.enabled", "true")
+
+			@data["ofelia"].each_with_index do |job, index|
+				name = job["name"] || "job-#{index}"
+				name = name.downcase.gsub(/[^a-z0-9\-]/, "-").gsub(/-+/, "-")
+				schedule = job["schedule"]
+				next unless schedule
+				command = job["command"]
+				next unless command
+				add_label("ofelia.job-exec.#{name}.schedule", schedule)
+				add_label("ofelia.job-exec.#{name}.command", command)
+				add_label("ofelia.job-exec.#{name}.user", job["user"]) if job["user"]
+				add_label("ofelia.job-exec.#{name}.tty", job["tty"].to_s) if job["tty"]!=nil
+				ol = job["no-overlap"]
+				add_label("ofelia.job-exec.#{name}.no-overlap", ol!=nil ? ol.to_s : "true")
+			end
+		end
+
 		MAPPINGS.each do |mapping|
 			yml_name, cmd_name, reverse_lambda, options = *mapping
 			next if $net_host && %w(links net ports).include?(yml_name)
@@ -266,7 +345,7 @@ class Container
 			end
 		end
 		
-		return cmd
+		return cmd.compact.join(" ")
 	end
 	
 	def build
@@ -281,7 +360,8 @@ class Container
 	end
 	
 	def stop_and_remove
-		puts "Stopping and removing old container..."
+		verbose "Stopping and removing old container..."
+		run_cmd("docker stop #{@name.shellescape} >/dev/null 2>&1", true)
 		run_cmd("docker rm -f #{@name.shellescape} >/dev/null 2>&1", true)
 	end
 	
@@ -290,9 +370,9 @@ class Container
 			networks = `docker network ls --format '{{.Name}}'`.split
 			@data["networks"].each do |network|
 				if networks.include?(network)
-					puts "Network #{network} exists."
+					verbose "Network #{network} exists."
 				else
-					puts "Creating network #{network}..."
+					verbose "Creating network #{network}..."
 					run_cmd("docker network create #{network.shellescape}", true)
 				end
 			end
@@ -302,14 +382,14 @@ class Container
 	def connect_to_networks
 		if !$net_host && @data["networks"].count>0
 			@data["networks"].each do |network|
-				puts "Connecting container to network #{network}..."
+				verbose "Connecting container to network #{network}..."
 				run_cmd("docker network connect #{network.shellescape} #{@name.shellescape} >/dev/null", true)
 			end
 		end
 	end
 	
 	def attach
-		puts "Attaching container..."
+		verbose "Attaching container..."
 		run_cmd("docker attach #{@name.shellescape}")
 	end
 	
@@ -332,17 +412,23 @@ class Container
 			self.pull
 		end
 		
+		if self.exists?
+			puts "Recreating #{@name}..."
+		else
+			puts "Creating   #{@name}..."
+		end
+		
 		self.stop_and_remove
 		
 		self.create_networks
 		
-		puts "Creating container..."
+		verbose "Creating container..."
 		cmd = build_run_command
-		run_cmd(cmd.compact.join(" ") + " >/dev/null")
+		run_cmd(cmd + " >/dev/null")
 		
 		self.connect_to_networks
 		
-		puts "Starting container..."
+		verbose "Starting container..."
 		run_cmd("docker start #{@name.shellescape} >/dev/null")
 		
 		(@data["after_run"] || []).each{|cmd| run_cmd(cmd)}
@@ -351,10 +437,12 @@ class Container
 			if ! @data["detach"]
 				self.attach
 			else
+				unless $quiet
 					self.log
 				end
 			end
 		end
+	end
 	
 	def regenerate
 		c_data = `docker inspect #{@name}`
@@ -375,24 +463,29 @@ class Container
 			data[name] = result
 		end
 		
+		@data = data
 		return data
 	end
 end
 
+def verbose(str); puts(str) if $verbose; end
+
 action = :run
 container = nil
 $dry_run = false
+$verbose = false
 $pull = false
 needs_container = true
 completion_str = ""
 silent_basedir = false
 $net_host = false
+$quiet = false
 
 if ENV['DUP_DIR']
 	$base_dir = ENV['DUP_DIR']
 else
 	$base_dir = File.join(Dir.home, ".dup")
-	puts "Environment variable DUP_DIR is not set. Looking for .yml files in #{base_dir}" unless silent_basedir
+	STDERR.puts "Environment variable DUP_DIR is not set. Looking for .yml files in #{$base_dir}" unless silent_basedir
 end
 
 opts = GetoptLong.new(
@@ -405,7 +498,10 @@ opts = GetoptLong.new(
 	[ '--update', '-u', GetoptLong::NO_ARGUMENT ],
 	[ '--_completion', GetoptLong::OPTIONAL_ARGUMENT ],
 	[ '--host', GetoptLong::NO_ARGUMENT ],
-	[ '--regenerate', GetoptLong::NO_ARGUMENT ]
+	[ '--regenerate', GetoptLong::NO_ARGUMENT ],
+	[ '--verbose', '-v', GetoptLong::NO_ARGUMENT ],
+	[ '--all', GetoptLong::NO_ARGUMENT ],
+	[ '--quiet', '-q', GetoptLong::NO_ARGUMENT ]
 )
 
 opts.each do |opt, arg|
@@ -436,10 +532,17 @@ opts.each do |opt, arg|
 			$net_host = true
 		when '--regenerate'
 			action = :regenerate
+			needs_container = false
+		when '--verbose'
+			$verbose = true
+		when '--all'
+			container = Dir[$base_dir + "/*.yml"].sort.map{|f| Container.new(File.basename(f, ".yml"))}
+		when '--quiet'
+			$quiet = true
 	end
 end
 
-container = ARGV.map{|c| Container.new(c)}
+container ||= ARGV.map{|c| Container.new(c)}
 
 if needs_container
 	if container.empty?
@@ -459,6 +562,7 @@ elsif action == :update
 	action_update(base_dir)
 elsif action == :regenerate
 	action_regenerate(container)
+	container.each {|c| puts c.build_run_command}
 elsif action == :_completion
 	action_completion($base_dir, completion_str)
 end

ofelias.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+require 'socket'
+require 'json'
+require 'terminal-table'
+
+def fetch_data
+    socket = UNIXSocket.new("/var/run/docker.sock")
+    request = "GET /v1.24/containers/json?all=1 HTTP/1.0\r\n\r\n"
+    socket.write(request)
+
+    response = ""
+
+    loop do
+            break if socket.eof?
+            line = socket.gets
+            break if line=="\r\n"
+    end
+
+    until socket.eof?
+            line = socket.gets
+            response += line
+    end
+
+    return JSON.parse(response, symbolize_names: true)
+end
+
+rows = []
+data = fetch_data
+data.each do |d|
+    next unless labels = d[:Labels]
+    name = d[:Names][0][1..-1]
+
+    next unless labels[:"ofelia.enabled"]
+    tasks = labels.keys.map(&:to_s).map{|s| s.scan(/\Aofelia\.job-([a-z]+)\.([^.]*)\./).first}.uniq.compact
+    next unless tasks.count>0
+    tasks.each do |task|
+        type, taskname = *task
+        schedule = labels[:"ofelia.job-#{type}.#{taskname}.schedule"]
+        command = labels[:"ofelia.job-#{type}.#{taskname}.command"]
+        rows << [name, type, taskname, schedule, command]
+    end
+end
+
+exit 0 unless rows.count>0
+
+puts Terminal::Table.new(rows: rows, headings: ["Container", "Type", "Task", "Schedule", "Command"])