Initial commit spam/ham scripts for dovecot

apache.cf

@@ -149,6 +149,8 @@ vars:
 	centos::
 		"modules" data => '[ 
 			{m:"proxy",f:"mod_proxy"},
+			{m:"proxy_http",f:"mod_proxy_http"},
+			{m:"proxy_http2",f:"mod_proxy_http2"},
 			{m:"proxy_fcgi",f:"mod_proxy_fcgi"},
 			{m: "autoindex",f:"mod_autoindex"},
 			{m: "log_config",f:"mod_log_config"},

dovecot.cf

@@ -6,6 +6,8 @@ bundle agent dovecot(cfg)
 {
 classes:
 	"use_ssl" expression => strcmp("$(cfg[ssl])","true");
+	"rspamd_scripts" expression => strcmp("$(cfg[rspamd_scripts])","true");
+
 vars:
 	freebsd::
 		"cfg_dir" string => "/usr/local/etc/dovecot";
@@ -75,6 +77,9 @@ methods:
 	"any" usebundle => wmde_install_packages(@(pkgs),"dovecot");
 	"any" usebundle => wmde_service("$(service_name)","dovecot_kept","dovecot_repaired"),
 		depends_on => @(service_deps);
+	rspamd_scripts::
+	"any" usebundle => dovecot_install_rspamd_sieve_scripts("$(dovecot.cfg_dir)/rspamd"),
+		depends_on => {"dovecot_pkgs_installed"};
 
 files:	
 	"$(cfg_dir)/."
@@ -166,6 +171,7 @@ bundle agent install_global_sieve_script( filename , content)
 {
 vars:
 	"rep_class" string =>"global_sieve_repaired_$(filename)";
+
 files:
 	"$(filename)"
 	create => "true",
@@ -175,12 +181,103 @@ files:
 	classes => if_repaired (sieve_repaired);
 
 commands:
-#	sieve_repaired::
-#		"$(dovecot.sievec_exe)"
-#		args => "$(filename)",
-#		depends_on => { "global_sieve_$(filename)_created" };
+	sieve_repaired::
+		"$(dovecot.sievec_exe)"
+		args => "$(filename)",
+		depends_on => { "global_sieve_$(filename)_created" };
+reports:
+
+}
+
+bundle agent dovecot_install_sieve_script( filename , content, cls)
+{
+vars:
+	"rep_class" string =>"global_sieve_$(filename)_repaired";
+
+files:
+	"$(filename)"
+	create => "true",
+	content => "$(content)",
+	perms => m("644"),
+	handle => "sieve_$(filename)_created",
+	classes => if_repaired ("$(cls)_repaired");
+
+commands:
+	"$(cls)_repaired"::
+		"$(dovecot.sievec_exe)"
+		args => "$(filename)",
+		depends_on => { "sieve_$(filename)_created" };
 reports:
 
 }
 
 
+
+
+
+bundle agent dovecot_rspamd_sieve_vars(read)
+{
+classes:
+	"$(read)";
+vars:
+#	do_read::
+#	"ham_script" string  => readfile("$(def.wmde_lib/scripts/rspamd-learn-ham.sieve.mustache)");
+#	"spam_script" string  => readfile("$(def.wmde_lib/scripts/rspamd-learn-ham.sieve.mustache)");
+#	"ham_sh" string  => readfile("$(def.wmde_lib/scripts/rspamd-learn-ham.sieve.mustache)");
+#	"spam_sh" string  => readfile("$(def.wmde_lib/scripts/rspamd-learn-ham.sieve.mustache)");
+
+}
+
+bundle agent dovecot_install_rspamd_sieve_scripts(target_dir)
+{
+methods:
+	"any" usebundle => dovecot_rspamd_sieve_vars("do_read"),
+		handle => "dovecot_rspamd_vars_ready";
+files:
+	"$(target_dir)/."
+	create => "true",
+	perms => m("755"),
+	handle => "dovecot_install_rspamd_sieve_scripts_dir_created";
+
+methods:
+	"any" usebundle => dovecot_install_sieve_script( "$(target_dir)/rspamd-learn-ham.sieve",
+#		string_mustache(
+			readfile("$(def.wmde_lib)/scripts/dovecot/rspamd-learn-ham.sieve.mustache")
+#			), 
+		, "dovecot_sieve_ham"
+	) ,depends_on => {"dovecot_install_rspamd_sieve_scripts_dir_created"};
+	"any" usebundle => dovecot_install_sieve_script( "$(target_dir)/rspamd-learn-spam.sieve",
+#		string_mustache(
+			readfile("$(def.wmde_lib)/scripts/dovecot/rspamd-learn-spam.sieve.mustache")
+#			)
+		, "dovecot_sieve_spam"
+		
+	) ,depends_on => {"dovecot_install_rspamd_sieve_scripts_dir_created"};
+
+files:
+
+	"$(target_dir)/rspamd-learn-ham.sh"
+	create => "true",
+	content => string_mustache(
+			readfile("$(def.wmde_lib)/scripts/dovecot/rspamd-learn-ham.sh.mustache")
+		),
+	perms => m("755"),
+	depends_on => {"dovecot_install_rspamd_sieve_scripts_dir_created"};
+	
+	"$(target_dir)/rspamd-learn-spam.sh"
+	create => "true",
+	content => string_mustache(
+			readfile("$(def.wmde_lib)/scripts/dovecot/rspamd-learn-spam.sh.mustache")
+		),
+	perms => m("755"),
+	depends_on => {"dovecot_install_rspamd_sieve_scripts_dir_created"};
+	
+
+
+reports:
+	"LOADER: $(def.wmde_lib)/scripts/dovecot/rspamd-learn-ham.sieve.mustache";
+	
+
+
+}
+

lib.cf

@@ -274,6 +274,7 @@ commands:
 	handle=>"system_repos_pkgs_installed";
 
 
+
 	centos_8::
 		"/usr/bin/dnf"
 		inform => "false",
@@ -283,6 +284,12 @@ commands:
 		"/usr/bin/dnf"
 		inform => "false",
 		args => "config-manager --set-enabled crb";
+
+		"/usr/bin/update-crypto-policies"
+		inform => "false",
+		contain => wmde_cmd_useshell,
+		args => "--set LEGACY > /dev/null";
+	
 methods:
 #	"any" usebundle => wmde_install_packages(@(pkgs),"system_repos");
 

php.cf

@@ -89,7 +89,9 @@ vars:
 			"php$(xversion)-intl",
 			"php$(xversion)-pecl-imagick-im6",
 			"php$(xversion)-ldap",
-			"php-pecl-json-post"
+			"php-pecl-json-post",
+			"php-pecl-apcu",
+			"php-pecl-zip",
 			
 		};
 
@@ -133,7 +135,7 @@ bundle agent php_install_centos_repos
 {
 classes:
 	centos::
-	"switch_repo" expression => not(returnszero("cat /etc/dnf/modules.d/php.module | grep -q '^stream.*=.*remi-$(php.version)' >/dev/null 2>&1","useshell"));
+	"switch_repo" expression => not(returnszero("cat /etc/dnf/modules.d/php.module 2>/dev/null | grep -q '^stream.*=.*remi-$(php.version)'","useshell"));
 	"install_remi" expression => not(returnszero("rpm -q remi-release > /dev/null","useshell"));
 	"install_epel" expression => not(returnszero("rpm -q epel-release > /dev/null","useshell"));
 

redis.cf

@@ -0,0 +1,22 @@
+#
+#
+#
+
+
+bundle agent redis
+{
+vars:
+	"pkgs" slist => {
+		"redis"
+	};
+	"service_name" string => "redis";
+	
+}
+
+
+bundle agent install_redis
+{
+methods:
+	"any" usebundle => wmde_install_packages(@(redis.pkgs),"redis");
+	"any" usebundle => wmde_service("$(redis.service_name)","redis_kept","redis_repaired");
+}

rspamd.cf

@@ -5,13 +5,16 @@
 bundle agent rspamd(cfg)
 {
 classes:
+	"runpw" expression => isvariable("cfg[password]");
+
 vars:
 	"cfgfiles" slist => {
 		"local.d/milter_headers.conf",
 #		"local.d/actions.conf",
       		"local.d/worker-normal.inc",
 		"local.d/worker-proxy.inc",
-#		"local.d/worker-controller.inc",
+		"local.d/worker-controller.inc",
+		"local.d/redis.conf",
 #		"local.d/classifier-bayes.conf",
 #		"local.d/worker-fuzzy.inc",
 #		"local.d/fuzzy_check.conf"
@@ -23,6 +26,7 @@ vars:
 		"service_name" string => "rspamd";
 		"root_user" string => "root";
 		"root_group" string => "wheel";
+		"rspamadm" string => "/usr/local/bin/rspamadm";
 
 	debian::
 		"pkgs" slist => {"rspamd"};
@@ -30,6 +34,7 @@ vars:
 		"service_name" string => "rspamd";
 		"root_user" string => "root";
 		"root_group" string => "root";
+		"rspamadm" string => "/usr//bin/rspamadm";
 
 	centos::
 		"pkgs" slist => {"rspamd"};
@@ -37,12 +42,17 @@ vars:
 		"service_name" string => "rspamd";
 		"root_user" string => "root";
 		"root_group" string => "root";
+		"rspamadm" string => "/usr//bin/rspamadm";
+
+	any::
+	"password" string => execresult("$(rspamadm) pw -p $(cfg[password])","noshell");
+
+	
 
 
 methods:
-	"any" usebundle => rspamd_install_yum_repo, handle => "rspamd_yum_repo_installed";
-	"any" usebundle => wmde_install_packages(@(pkgs),"rspamd"),
-		depends_on => {"rspamd_yum_repo_installed"};
+	"any" usebundle => install_rspamd_pkgs;
+
 	"any" usebundle => wmde_service("$(service_name)","rspamd_kept","rspamd_repaired"),
 		depends_on => {"rspamd_cfgs_done"};
 files:
@@ -61,6 +71,8 @@ files:
 	classes => if_repaired(rspamd_repaired),
 	handle => "rspamd_cfgs_done";
 
+
+
 reports:
 #	"RSPAMD: $(cfgjs)";
 #	"RSJ: $(worker_normalx)";
@@ -81,3 +93,12 @@ commands:
 
 
 }
+
+
+bundle agent install_rspamd_pkgs
+{
+methods:
+	"any" usebundle => rspamd_install_yum_repo, handle => "rspamd_yum_repo_installed";
+	"any" usebundle => wmde_install_packages(@(rspamd.pkgs),"rspamd"),
+		depends_on => {"rspamd_yum_repo_installed"};
+}

scripts/dovecot/rspamd-learn-ham.sh.mustache

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/bin/rspamc -h localhost:11334 learn_ham

scripts/dovecot/rspamd-learn-ham.sieve.mustache

@@ -0,0 +1,16 @@
+require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"];
+  
+if environment :matches "imap.mailbox" "*" {
+  set "mailbox" "${1}";
+}
+
+if string "${mailbox}" "Trash" {
+  stop;
+}
+
+if environment :matches "imap.user" "*" {
+  set "username" "${1}";
+}
+
+pipe :copy "rspamd-learn-ham.sh" [ "${username}" ];
+

scripts/dovecot/rspamd-learn-spam.sh.mustache

@@ -0,0 +1,3 @@
+#!/bin/sh
+exec /usr/bin/rspamc -h localhost:11334 learn_spam
+

scripts/dovecot/rspamd-learn-spam.sieve.mustache

@@ -0,0 +1,8 @@
+require ["vnd.dovecot.pipe", "copy", "imapsieve", "environment", "variables"];
+  
+if environment :matches "imap.user" "*" {
+  set "username" "${1}";
+}
+
+pipe :copy "rspamd-learn-spam.sh" [ "${username}" ];
+

sympa.cf

@@ -284,7 +284,7 @@ reports:
 }
 
 
-bundle agent install_sympa_domain(domain,data)
+bundle agent install_sympa_domain(domain,data,conf_dir)
 {
 vars: 
 	"default_settings" data => '{
@@ -297,16 +297,16 @@ vars:
 	"settings[domain]" string => "$(domain)";
 
 files:
-	"$(sympa.conf_dir)/$(domain)/."
+	"$(conf_dir)/$(domain)/."
 		create => "true",
 		perms => m("755");
 
-	"$(sympa.conf_dir)/$(domain)/robot.conf"
+	"$(conf_dir)/$(domain)/robot.conf"
 		perms => m("644"),
 		copy_from => seed_cp("$(sys.workdir)/inputs/$(def.wmde_libdir)/templates/sympa/robot.conf"),
 		handle => "sympa_robot_$(domain)_ready";
 
-	"$(sympa.conf_dir)/$(domain)/robot.conf"
+	"$(conf_dir)/$(domain)/robot.conf"
 		create => "true",
 		perms => m("644"),  #"sympa","sympa"),
 		edit_line => set_config_values("$(this.bundle).settings"),
@@ -321,12 +321,12 @@ reports:
 #	"JO: $(jo)";
 }
 
-bundle agent install_sympa_domains(dl)
+bundle agent install_sympa_domains(dl,conf_dir)
 {
 vars:
 	"idx" slist => getindices(@(dl));
 methods:
-	"any" usebundle => install_sympa_domain("$(idx)",@(dl[$(idx)]));
+	"any" usebundle => install_sympa_domain("$(idx)",@(dl[$(idx)]),"$(conf_dir)");
 
 reports:
 }
@@ -346,10 +346,18 @@ files:
 
 bundle agent install_sympa(cfg)
 {
+classes:
+	"etc_changed" expression => isvariable("cfg[settings][etc]");
+
 vars:
-#	"cfg_domains" data => @(cfg[domains]);
 	"domains_idx" slist => getindices(@(cfg[domains]));
-#	"cfg_domains" slist => {"a","b","c"};
+
+	etc_changed::
+		"cfg_dir" string => "$(cfg[settings][etc])";
+
+	!etc_changed::
+		"cfg_dir" string => "$(sympa.conf_dir)";
+
 methods:
 	"any" usebundle => sympa_create_mailname(@(cfg));
 	"any" usebundle => wmde_install_packages(@(sympa.pkgs),"sympa");
@@ -359,7 +367,8 @@ methods:
 	"any" usebundle => wmde_service("$(sympa.service_name)","sympa_kept","sympa_repaired"),
 		depends_on => {
 			"sympa_pkgs_installed",
-			"sympa_config_updated"
+			"sympa_config_updated",
+			"sympa_robots_created"
 		};
 	"any" usebundle => wmde_service("$(sympa.wwservice_name)","sympa_kept","sympa_repaired"),
 		depends_on => {
@@ -368,11 +377,10 @@ methods:
 			"sympa_robots_created"
 		};
 
-#	"any" usebundle => install_sympa_domain("@(cfg[domains][$(domains_idx)])");
-	"any" usebundle => install_sympa_domains(@(cfg[domains]));
+	"any" usebundle => install_sympa_domains(@(cfg[domains]),"$(cfg_dir)");
 reports:
-#	"IDX $(domains_idx)";
-#	"DOMCONF: $(cfg[$(idx)])";
+	"ETC: $(cfg_dir) - $(cfg[settings][etc])";
+
 }
 
 

templates/dovecot/dovecot.conf.mustache

@@ -95,6 +95,56 @@ service {{@}} {
 
 {{/vars.dovecot.cfg.services}}
 
+{{#vars.dovecot.cfg.default_imap_folders}}
+
+namespace inbox {
+	inbox = yes
+	# These mailboxes are widely used and could perhaps be created automatically:
+	mailbox Drafts {
+	special_use = \Drafts
+		auto = subscribe
+	}
+	mailbox Junk {
+		auto = subscribe
+		special_use = \Junk
+	}
+	mailbox Trash {
+		auto = subscribe
+		special_use = \Trash
+	}
+
+	mailbox Sent {
+		auto = subscribe
+		special_use = \Sent
+	}
+}
+{{/vars.dovecot.cfg.default_imap_folders}}
+
+{{#vars.dovecot.cfg.rspamd_scripts}}
+	protocol imap {
+		mail_plugins = $mail_plugins imap_sieve
+	}
+	plugin {
+		sieve_plugins = sieve_imapsieve sieve_extprograms 
+
+		# From elsewhere to Spam folder
+		imapsieve_mailbox1_name = Junk
+		imapsieve_mailbox1_causes = COPY
+		imapsieve_mailbox1_before = file:/etc/dovecot/rspamd/rspamd-learn-spam.sieve
+
+		# From Spam folder to elsewhere
+		imapsieve_mailbox2_name = *
+		imapsieve_mailbox2_from = Junk
+		imapsieve_mailbox2_causes = COPY
+		imapsieve_mailbox2_before = file:/etc/dovecot/rspamd/rspamd-learn-ham.sieve
+
+		sieve_pipe_bin_dir = /etc/dovecot/rspamd
+
+		sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.environment
+	}
+{{/vars.dovecot.cfg.rspamd_scripts}}
+
+
 
 
 {{vars.dovecot.cfg.raw}}