src/automod/AIAutoModService.ts

/**
 * This file is part of SudoBot.
 *
 * Copyright (C) 2021-2023 OSN Developers.
 *
 * SudoBot is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * SudoBot is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with SudoBot. If not, see <https://www.gnu.org/licenses/>.
 */

import { Message, PermissionFlagsBits } from "discord.js";
import { google } from "googleapis";
import Service from "../core/Service";
import { HasEventListeners } from "../types/HasEventListeners";
import { log, logError } from "../utils/logger";
import { isImmuneToAutoMod } from "../utils/utils";

export const name = "aiAutoMod";
const discoveryURL = "https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1";

// TODO: Add support for other type of message attributes

export default class AIAutoModService extends Service implements HasEventListeners {
    protected googleClient: any = undefined;

    analyze(client: any, params: any) {
        return new Promise<any>((resolve, reject) => {
            client.comments.analyze(params, (error: any, response: any) => {
                if (error) {
                    reject(error);
                    return;
                }

                resolve(response);
            });
        });
    }

    async boot() {
        this.googleClient = await google.discoverAPI<any>(discoveryURL);
    }

    async onMessageCreate(message: Message<boolean>) {
        if (!process.env.PERSPECTIVE_API_TOKEN || !message.content?.trim()) {
            return false;
        }

        const config = this.client.configManager.config[message.guildId!]?.ai_automod;

        if (!config?.enabled || !message.deletable) {
            return false;
        }

        if (await isImmuneToAutoMod(this.client, message.member!, PermissionFlagsBits.ManageMessages)) {
            return;
        }

        const {
            max_severe_toxicity,
            max_threat,
            max_toxicity,
            max_explicit,
            max_flirtation,
            max_identity_attack,
            max_insult,
            max_profanity
        } = config.parameters;

        try {
            const response = await this.analyze(this.googleClient, {
                key: process.env.PERSPECTIVE_API_TOKEN,
                resource: {
                    requestedAttributes: {
                        TOXICITY: {},
                        THREAT: {},
                        SEVERE_TOXICITY: {},
                        IDENTITY_ATTACK: {},
                        INSULT: {},
                        PROFANITY: {},
                        SEXUALLY_EXPLICIT: {},
                        FLIRTATION: {}
                    },
                    comment: {
                        text: message.content
                    },
                    languages: ["en"]
                }
            });

            log(JSON.stringify(response.data.attributeScores, null, 4));

            const threatScore = response.data.attributeScores.THREAT.summaryScore.value * 100;
            const toxicityScore = response.data.attributeScores.TOXICITY.summaryScore.value * 100;
            const severeToxicityScore = response.data.attributeScores.SEVERE_TOXICITY.summaryScore.value * 100;
            const explicitScore = response.data.attributeScores.SEXUALLY_EXPLICIT.summaryScore.value * 100;
            const flirtationScore = response.data.attributeScores.FLIRTATION.summaryScore.value * 100;
            const identityAttackScore = response.data.attributeScores.IDENTITY_ATTACK.summaryScore.value * 100;
            const insultScore = response.data.attributeScores.INSULT.summaryScore.value * 100;
            const profanityScore = response.data.attributeScores.PROFANITY.summaryScore.value * 100;

            const isThreat = threatScore >= max_threat;
            const isToxic = toxicityScore >= max_toxicity;
            const isSeverelyToxic = severeToxicityScore >= max_severe_toxicity;
            const isExplicit = explicitScore >= max_explicit;
            const isFlirty = flirtationScore >= max_flirtation;
            const isAttack = identityAttackScore >= max_identity_attack;
            const isInsult = insultScore >= max_insult;
            const isProfanity = profanityScore >= max_profanity;

            if (isThreat || isToxic || isSeverelyToxic || isExplicit || isFlirty || isAttack || isInsult || isProfanity) {
                await message.delete();
                await this.client.logger.logAIAutoModMessageDelete({
                    message,
                    toxicityScore,
                    severeToxicityScore,
                    threatScore,
                    isSeverelyToxic,
                    isThreat,
                    isToxic,
                    isExplicit,
                    isFlirty,
                    isAttack,
                    isInsult,
                    isProfanity,
                    explicitScore,
                    flirtationScore,
                    identityAttackScore,
                    insultScore,
                    profanityScore
                });
            }
        } catch (e) {
            logError(e);
        }
    }
}
1	/**
2	* This file is part of SudoBot.
3	*
4	* Copyright (C) 2021-2023 OSN Developers.
5	*
6	* SudoBot is free software; you can redistribute it and/or modify it
7	* under the terms of the GNU Affero General Public License as published by
8	* the Free Software Foundation, either version 3 of the License, or
9	* (at your option) any later version.
10	*
11	* SudoBot is distributed in the hope that it will be useful, but
12	* WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	* GNU Affero General Public License for more details.
15	*
16	* You should have received a copy of the GNU Affero General Public License
17	* along with SudoBot. If not, see <https://www.gnu.org/licenses/>.
18	*/
19
20	import { Message, PermissionFlagsBits } from "discord.js";
21	import { google } from "googleapis";
22	import Service from "../core/Service";
23	import { HasEventListeners } from "../types/HasEventListeners";
24	import { log, logError } from "../utils/logger";
25	import { isImmuneToAutoMod } from "../utils/utils";
26
27	export const name = "aiAutoMod";
28	const discoveryURL = "https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1";
29
30	// TODO: Add support for other type of message attributes
31
32	export default class AIAutoModService extends Service implements HasEventListeners {
33	protected googleClient: any = undefined;
34
35	analyze(client: any, params: any) {
36	return new Promise<any>((resolve, reject) => {
37	client.comments.analyze(params, (error: any, response: any) => {
38	if (error) {
39	reject(error);
40	return;
41	}
42
43	resolve(response);
44	});
45	});
46	}
47
48	async boot() {
49	this.googleClient = await google.discoverAPI<any>(discoveryURL);
50	}
51
52	async onMessageCreate(message: Message<boolean>) {
53	if (!process.env.PERSPECTIVE_API_TOKEN \|\| !message.content?.trim()) {
54	return false;
55	}
56
57	const config = this.client.configManager.config[message.guildId!]?.ai_automod;
58
59	if (!config?.enabled \|\| !message.deletable) {
60	return false;
61	}
62
63	if (await isImmuneToAutoMod(this.client, message.member!, PermissionFlagsBits.ManageMessages)) {
64	return;
65	}
66
67	const {
68	max_severe_toxicity,
69	max_threat,
70	max_toxicity,
71	max_explicit,
72	max_flirtation,
73	max_identity_attack,
74	max_insult,
75	max_profanity
76	} = config.parameters;
77
78	try {
79	const response = await this.analyze(this.googleClient, {
80	key: process.env.PERSPECTIVE_API_TOKEN,
81	resource: {
82	requestedAttributes: {
83	TOXICITY: {},
84	THREAT: {},
85	SEVERE_TOXICITY: {},
86	IDENTITY_ATTACK: {},
87	INSULT: {},
88	PROFANITY: {},
89	SEXUALLY_EXPLICIT: {},
90	FLIRTATION: {}
91	},
92	comment: {
93	text: message.content
94	},
95	languages: ["en"]
96	}
97	});
98
99	log(JSON.stringify(response.data.attributeScores, null, 4));
100
101	const threatScore = response.data.attributeScores.THREAT.summaryScore.value * 100;
102	const toxicityScore = response.data.attributeScores.TOXICITY.summaryScore.value * 100;
103	const severeToxicityScore = response.data.attributeScores.SEVERE_TOXICITY.summaryScore.value * 100;
104	const explicitScore = response.data.attributeScores.SEXUALLY_EXPLICIT.summaryScore.value * 100;
105	const flirtationScore = response.data.attributeScores.FLIRTATION.summaryScore.value * 100;
106	const identityAttackScore = response.data.attributeScores.IDENTITY_ATTACK.summaryScore.value * 100;
107	const insultScore = response.data.attributeScores.INSULT.summaryScore.value * 100;
108	const profanityScore = response.data.attributeScores.PROFANITY.summaryScore.value * 100;
109
110	const isThreat = threatScore >= max_threat;
111	const isToxic = toxicityScore >= max_toxicity;
112	const isSeverelyToxic = severeToxicityScore >= max_severe_toxicity;
113	const isExplicit = explicitScore >= max_explicit;
114	const isFlirty = flirtationScore >= max_flirtation;
115	const isAttack = identityAttackScore >= max_identity_attack;
116	const isInsult = insultScore >= max_insult;
117	const isProfanity = profanityScore >= max_profanity;
118
119	if (isThreat \|\| isToxic \|\| isSeverelyToxic \|\| isExplicit \|\| isFlirty \|\| isAttack \|\| isInsult \|\| isProfanity) {
120	await message.delete();
121	await this.client.logger.logAIAutoModMessageDelete({
122	message,
123	toxicityScore,
124	severeToxicityScore,
125	threatScore,
126	isSeverelyToxic,
127	isThreat,
128	isToxic,
129	isExplicit,
130	isFlirty,
131	isAttack,
132	isInsult,
133	isProfanity,
134	explicitScore,
135	flirtationScore,
136	identityAttackScore,
137	insultScore,
138	profanityScore
139	});
140	}
141	} catch (e) {
142	logError(e);
143	}
144	}
145	}