From 5f84403175673899506b6b3650c3ccb7ac2e99c7 Mon Sep 17 00:00:00 2001 From: Zabuzard Date: Mon, 25 Nov 2024 10:15:11 +0100 Subject: [PATCH 1/2] Revert "Removed now obsolete HelperPrune routine (config change) (#1076)" This reverts commit a11088e7e5aa65b519edb3ee966d14164223935c. --- application/config.json.template | 7 + .../org/togetherjava/tjbot/config/Config.java | 13 ++ .../tjbot/config/HelperPruneConfig.java | 19 ++ .../togetherjava/tjbot/features/Features.java | 2 + .../features/help/AutoPruneHelperRoutine.java | 191 ++++++++++++++++++ 5 files changed, 232 insertions(+) create mode 100644 application/src/main/java/org/togetherjava/tjbot/config/HelperPruneConfig.java create mode 100644 application/src/main/java/org/togetherjava/tjbot/features/help/AutoPruneHelperRoutine.java diff --git a/application/config.json.template b/application/config.json.template index b5b591eff1..bc679ff53a 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -98,6 +98,13 @@ "rateLimitWindowSeconds": 10, "rateLimitRequestsInWindow": 3 }, + "helperPruneConfig": { + "roleFullLimit": 100, + "roleFullThreshold": 95, + "pruneMemberAmount": 7, + "inactivateAfterDays": 90, + "recentlyJoinedDays": 4 + }, "featureBlacklist": { "normal": [ ], diff --git a/application/src/main/java/org/togetherjava/tjbot/config/Config.java b/application/src/main/java/org/togetherjava/tjbot/config/Config.java index 5d46cae584..79c04e6cad 100644 --- a/application/src/main/java/org/togetherjava/tjbot/config/Config.java +++ b/application/src/main/java/org/togetherjava/tjbot/config/Config.java @@ -43,6 +43,7 @@ public final class Config { private final String openaiApiKey; private final String sourceCodeBaseUrl; private final JShellConfig jshell; + private final HelperPruneConfig helperPruneConfig; private final FeatureBlacklistConfig featureBlacklistConfig; private final RSSFeedsConfig rssFeedsConfig; private final String selectRolesChannelPattern; @@ -93,6 +94,8 @@ private Config(@JsonProperty(value = "token", required = true) String token, @JsonProperty(value = "jshell", required = true) JShellConfig jshell, @JsonProperty(value = "memberCountCategoryPattern", required = true) String memberCountCategoryPattern, + @JsonProperty(value = "helperPruneConfig", + required = true) HelperPruneConfig helperPruneConfig, @JsonProperty(value = "featureBlacklist", required = true) FeatureBlacklistConfig featureBlacklistConfig, @JsonProperty(value = "rssConfig", required = true) RSSFeedsConfig rssFeedsConfig, @@ -128,6 +131,7 @@ private Config(@JsonProperty(value = "token", required = true) String token, this.openaiApiKey = Objects.requireNonNull(openaiApiKey); this.sourceCodeBaseUrl = Objects.requireNonNull(sourceCodeBaseUrl); this.jshell = Objects.requireNonNull(jshell); + this.helperPruneConfig = Objects.requireNonNull(helperPruneConfig); this.featureBlacklistConfig = Objects.requireNonNull(featureBlacklistConfig); this.rssFeedsConfig = Objects.requireNonNull(rssFeedsConfig); this.selectRolesChannelPattern = Objects.requireNonNull(selectRolesChannelPattern); @@ -396,6 +400,15 @@ public JShellConfig getJshell() { return jshell; } + /** + * Gets the config for automatic pruning of helper roles. + * + * @return the configuration + */ + public HelperPruneConfig getHelperPruneConfig() { + return helperPruneConfig; + } + /** * The configuration of blacklisted features. * diff --git a/application/src/main/java/org/togetherjava/tjbot/config/HelperPruneConfig.java b/application/src/main/java/org/togetherjava/tjbot/config/HelperPruneConfig.java new file mode 100644 index 0000000000..6f451b491f --- /dev/null +++ b/application/src/main/java/org/togetherjava/tjbot/config/HelperPruneConfig.java @@ -0,0 +1,19 @@ +package org.togetherjava.tjbot.config; + + +/** + * Config for automatic pruning of helper roles, see + * {@link org.togetherjava.tjbot.features.help.AutoPruneHelperRoutine}. + * + * @param roleFullLimit if a helper role contains that many users, it is considered full and pruning + * must occur + * @param roleFullThreshold if a helper role contains that many users, pruning will start to occur + * to prevent reaching the limit + * @param pruneMemberAmount amount of users to remove from helper roles during a prune + * @param inactivateAfterDays after how many days of inactivity a user is eligible for pruning + * @param recentlyJoinedDays if a user is with the server for just this amount of days, they are + * protected from pruning + */ +public record HelperPruneConfig(int roleFullLimit, int roleFullThreshold, int pruneMemberAmount, + int inactivateAfterDays, int recentlyJoinedDays) { +} diff --git a/application/src/main/java/org/togetherjava/tjbot/features/Features.java b/application/src/main/java/org/togetherjava/tjbot/features/Features.java index 82b1e327e1..72c484f032 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/Features.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/Features.java @@ -132,6 +132,8 @@ public static Collection createFeatures(JDA jda, Database database, Con features.add(new ScamHistoryPurgeRoutine(scamHistoryStore)); features.add(new HelpThreadMetadataPurger(database)); features.add(new HelpThreadActivityUpdater(helpSystemHelper)); + features + .add(new AutoPruneHelperRoutine(config, helpSystemHelper, modAuditLogWriter, database)); features.add(new HelpThreadAutoArchiver(helpSystemHelper)); features.add(new LeftoverBookmarksCleanupRoutine(bookmarksSystem)); features.add(new MarkHelpThreadCloseInDBRoutine(database, helpThreadLifecycleListener)); diff --git a/application/src/main/java/org/togetherjava/tjbot/features/help/AutoPruneHelperRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/help/AutoPruneHelperRoutine.java new file mode 100644 index 0000000000..e32547e4f8 --- /dev/null +++ b/application/src/main/java/org/togetherjava/tjbot/features/help/AutoPruneHelperRoutine.java @@ -0,0 +1,191 @@ +package org.togetherjava.tjbot.features.help; + +import net.dv8tion.jda.api.JDA; +import net.dv8tion.jda.api.entities.Guild; +import net.dv8tion.jda.api.entities.Member; +import net.dv8tion.jda.api.entities.Role; +import net.dv8tion.jda.api.entities.channel.concrete.TextChannel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.togetherjava.tjbot.config.Config; +import org.togetherjava.tjbot.config.HelperPruneConfig; +import org.togetherjava.tjbot.db.Database; +import org.togetherjava.tjbot.features.Routine; +import org.togetherjava.tjbot.features.moderation.audit.ModAuditLogWriter; + +import javax.annotation.Nullable; + +import java.time.Duration; +import java.time.Instant; +import java.time.Period; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.regex.Pattern; + +import static org.togetherjava.tjbot.db.generated.tables.HelpChannelMessages.HELP_CHANNEL_MESSAGES; + +/** + * Due to a technical limitation in Discord, roles with more than 100 users can not be ghost-pinged + * into helper threads. + *

+ * This routine mitigates the problem by automatically pruning inactive users from helper roles + * approaching this limit. + */ +public final class AutoPruneHelperRoutine implements Routine { + private static final Logger logger = LoggerFactory.getLogger(AutoPruneHelperRoutine.class); + + private final int roleFullLimit; + private final int roleFullThreshold; + private final int pruneMemberAmount; + private final Period inactiveAfter; + private final int recentlyJoinedDays; + + private final HelpSystemHelper helper; + private final ModAuditLogWriter modAuditLogWriter; + private final Database database; + private final List allCategories; + private final Predicate selectYourRolesChannelNamePredicate; + + /** + * Creates a new instance. + * + * @param config to determine all helper categories + * @param helper the helper to use + * @param modAuditLogWriter to inform mods when manual pruning becomes necessary + * @param database to determine whether an user is inactive + */ + public AutoPruneHelperRoutine(Config config, HelpSystemHelper helper, + ModAuditLogWriter modAuditLogWriter, Database database) { + allCategories = config.getHelpSystem().getCategories(); + this.helper = helper; + this.modAuditLogWriter = modAuditLogWriter; + this.database = database; + + HelperPruneConfig helperPruneConfig = config.getHelperPruneConfig(); + roleFullLimit = helperPruneConfig.roleFullLimit(); + roleFullThreshold = helperPruneConfig.roleFullThreshold(); + pruneMemberAmount = helperPruneConfig.pruneMemberAmount(); + inactiveAfter = Period.ofDays(helperPruneConfig.inactivateAfterDays()); + recentlyJoinedDays = helperPruneConfig.recentlyJoinedDays(); + selectYourRolesChannelNamePredicate = + Pattern.compile(config.getSelectRolesChannelPattern()).asMatchPredicate(); + } + + @Override + public Schedule createSchedule() { + return new Schedule(ScheduleMode.FIXED_RATE, 0, 1, TimeUnit.HOURS); + } + + @Override + public void runRoutine(JDA jda) { + jda.getGuildCache().forEach(this::pruneForGuild); + } + + private void pruneForGuild(Guild guild) { + Instant now = Instant.now(); + TextChannel selectRoleChannel = getSelectRolesChannelOptional(guild.getJDA()).orElse(null); + + allCategories.stream() + .map(category -> helper.handleFindRoleForCategory(category, guild)) + .filter(Optional::isPresent) + .map(Optional::orElseThrow) + .forEach(role -> pruneRoleIfFull(role, selectRoleChannel, now)); + } + + private void pruneRoleIfFull(Role role, @Nullable TextChannel selectRoleChannel, Instant when) { + role.getGuild().findMembersWithRoles(role).onSuccess(members -> { + if (isRoleFull(members)) { + logger.debug("Helper role {} is full, starting to prune.", role.getName()); + pruneRole(role, members, selectRoleChannel, when); + } + }); + } + + private boolean isRoleFull(Collection members) { + return members.size() >= roleFullThreshold; + } + + private void pruneRole(Role role, List members, + @Nullable TextChannel selectRoleChannel, Instant when) { + List membersShuffled = new ArrayList<>(members); + Collections.shuffle(membersShuffled); + + List membersToPrune = membersShuffled.stream() + .filter(member -> isMemberInactive(member, when)) + .limit(pruneMemberAmount) + .toList(); + if (membersToPrune.size() < pruneMemberAmount) { + warnModsAbout( + "Attempting to prune helpers from role **%s** (%d members), but only found %d inactive users. That is less than expected, the category might eventually grow beyond the limit." + .formatted(role.getName(), members.size(), membersToPrune.size()), + role.getGuild()); + } + if (members.size() - membersToPrune.size() >= roleFullLimit) { + warnModsAbout( + "The helper role **%s** went beyond its member limit (%d), despite automatic pruning. It will not function correctly anymore. Please manually prune some users." + .formatted(role.getName(), roleFullLimit), + role.getGuild()); + } + + logger.info("Pruning {} users {} from role {}", membersToPrune.size(), membersToPrune, + role.getName()); + membersToPrune.forEach(member -> pruneMemberFromRole(member, role, selectRoleChannel)); + } + + private boolean isMemberInactive(Member member, Instant when) { + if (member.hasTimeJoined()) { + Instant memberJoined = member.getTimeJoined().toInstant(); + if (Duration.between(memberJoined, when).toDays() <= recentlyJoinedDays) { + // New users are protected from purging to not immediately kick them out of the role + // again + return false; + } + } + + Instant latestActiveMoment = when.minus(inactiveAfter); + + // Has no recent help message + return database.read(context -> context.fetchCount(HELP_CHANNEL_MESSAGES, + HELP_CHANNEL_MESSAGES.GUILD_ID.eq(member.getGuild().getIdLong()) + .and(HELP_CHANNEL_MESSAGES.AUTHOR_ID.eq(member.getIdLong())) + .and(HELP_CHANNEL_MESSAGES.SENT_AT.greaterThan(latestActiveMoment)))) == 0; + } + + private void pruneMemberFromRole(Member member, Role role, + @Nullable TextChannel selectRoleChannel) { + Guild guild = member.getGuild(); + + String channelMentionOrFallbackMessage = + selectRoleChannel == null ? "role selection channel" + : selectRoleChannel.getAsMention(); + + String dmMessage = + """ + You seem to have been inactive for some time in server **%s**, hence we removed you from the **%s** role. + If that was a mistake, just head back to %s and select the role again. + Sorry for any inconvenience caused by this 🙇""" + .formatted(guild.getName(), role.getName(), channelMentionOrFallbackMessage); + + guild.removeRoleFromMember(member, role) + .flatMap(any -> member.getUser().openPrivateChannel()) + .flatMap(channel -> channel.sendMessage(dmMessage)) + .queue(null, failure -> logger.debug( + "Failed sending a DM to user ({}) while pruning them from a helper role.", + member.getId())); + } + + private void warnModsAbout(String message, Guild guild) { + logger.warn(message); + + modAuditLogWriter.write("Auto-prune helpers", message, null, Instant.now(), guild); + } + + private Optional getSelectRolesChannelOptional(JDA jda) { + return jda.getTextChannels() + .stream() + .filter(textChannel -> selectYourRolesChannelNamePredicate.test(textChannel.getName())) + .findFirst(); + } +} From 49bf6fbc12b7830161660bbc618dec811496f8f6 Mon Sep 17 00:00:00 2001 From: Zabuzard Date: Mon, 25 Nov 2024 10:20:58 +0100 Subject: [PATCH 2/2] bumped role limit from 100 to 250 --- application/config.json.template | 4 ++-- .../main/java/org/togetherjava/tjbot/features/Features.java | 1 + .../tjbot/features/help/AutoPruneHelperRoutine.java | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/application/config.json.template b/application/config.json.template index bc679ff53a..c34d9e8635 100644 --- a/application/config.json.template +++ b/application/config.json.template @@ -99,8 +99,8 @@ "rateLimitRequestsInWindow": 3 }, "helperPruneConfig": { - "roleFullLimit": 100, - "roleFullThreshold": 95, + "roleFullLimit": 250, + "roleFullThreshold": 245, "pruneMemberAmount": 7, "inactivateAfterDays": 90, "recentlyJoinedDays": 4 diff --git a/application/src/main/java/org/togetherjava/tjbot/features/Features.java b/application/src/main/java/org/togetherjava/tjbot/features/Features.java index 72c484f032..99241f5689 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/Features.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/Features.java @@ -23,6 +23,7 @@ import org.togetherjava.tjbot.features.filesharing.FileSharingMessageListener; import org.togetherjava.tjbot.features.github.GitHubCommand; import org.togetherjava.tjbot.features.github.GitHubReference; +import org.togetherjava.tjbot.features.help.AutoPruneHelperRoutine; import org.togetherjava.tjbot.features.help.GuildLeaveCloseThreadListener; import org.togetherjava.tjbot.features.help.HelpSystemHelper; import org.togetherjava.tjbot.features.help.HelpThreadActivityUpdater; diff --git a/application/src/main/java/org/togetherjava/tjbot/features/help/AutoPruneHelperRoutine.java b/application/src/main/java/org/togetherjava/tjbot/features/help/AutoPruneHelperRoutine.java index e32547e4f8..b04e01776a 100644 --- a/application/src/main/java/org/togetherjava/tjbot/features/help/AutoPruneHelperRoutine.java +++ b/application/src/main/java/org/togetherjava/tjbot/features/help/AutoPruneHelperRoutine.java @@ -27,7 +27,7 @@ import static org.togetherjava.tjbot.db.generated.tables.HelpChannelMessages.HELP_CHANNEL_MESSAGES; /** - * Due to a technical limitation in Discord, roles with more than 100 users can not be ghost-pinged + * Due to a technical limitation in Discord, roles with more than 250 users cannot be ghost-pinged * into helper threads. *

* This routine mitigates the problem by automatically pruning inactive users from helper roles @@ -54,7 +54,7 @@ public final class AutoPruneHelperRoutine implements Routine { * @param config to determine all helper categories * @param helper the helper to use * @param modAuditLogWriter to inform mods when manual pruning becomes necessary - * @param database to determine whether an user is inactive + * @param database to determine whether a user is inactive */ public AutoPruneHelperRoutine(Config config, HelpSystemHelper helper, ModAuditLogWriter modAuditLogWriter, Database database) {