feat: isolate profanity filter with separate database

- Create dedicated ProfanityFilter class with isolated SQLite database
- Separate profanity.db from main application database to prevent SQLITE_MISUSE errors
- Add comprehensive custom word management (CRUD operations)
- Implement advanced profanity detection with leetspeak and pattern matching
- Add admin UI for managing custom profanity words
- Add extensive test suites for both profanity filter and API routes
- Update server.js to use isolated profanity filter
- Add proper database initialization and cleanup methods
- Support in-memory databases for testing

Breaking changes:
- Profanity filter now uses separate database file
- Updated admin API endpoints for profanity management
- Enhanced profanity detection capabilities
This commit is contained in:
Deco Vander 2025-07-04 00:03:24 -04:00
parent d19cd2766c
commit c7f39e4939
15 changed files with 5365 additions and 257 deletions

View file

@ -0,0 +1,325 @@
const sqlite3 = require('sqlite3').verbose();
const ProfanityFilter = require('../profanity-filter');
describe('ProfanityFilter', () => {
let filter;
beforeEach(async () => {
// Create profanity filter with in-memory database
filter = new ProfanityFilter(':memory:');
// Wait for initialization to complete
await new Promise(resolve => setTimeout(resolve, 200));
// Ensure custom words are loaded
await filter.loadCustomWords();
});
afterEach(() => {
if (filter) {
filter.close();
}
});
describe('Basic Profanity Detection', () => {
test('should detect single profanity word', () => {
const result = filter.analyzeProfanity('This is shit');
expect(result.hasProfanity).toBe(true);
expect(result.count).toBe(1);
expect(result.severity).toBe('medium');
expect(result.matches[0].word).toBe('shit');
});
test('should detect multiple profanity words', () => {
const result = filter.analyzeProfanity('This fucking shit is damn terrible');
expect(result.hasProfanity).toBe(true);
expect(result.count).toBe(3);
expect(result.severity).toBe('medium');
expect(result.matches.map(m => m.word)).toContain('fuck');
expect(result.matches.map(m => m.word)).toContain('shit');
expect(result.matches.map(m => m.word)).toContain('damn');
});
test('should not detect profanity in clean text', () => {
const result = filter.analyzeProfanity('This road is very slippery with ice');
expect(result.hasProfanity).toBe(false);
expect(result.count).toBe(0);
expect(result.severity).toBe('none');
expect(result.matches).toHaveLength(0);
});
test('should handle empty or null input', () => {
expect(filter.analyzeProfanity('')).toEqual({
hasProfanity: false,
matches: [],
severity: 'none',
count: 0,
filtered: ''
});
expect(filter.analyzeProfanity(null)).toEqual({
hasProfanity: false,
matches: [],
severity: 'none',
count: 0,
filtered: null
});
expect(filter.analyzeProfanity(undefined)).toEqual({
hasProfanity: false,
matches: [],
severity: 'none',
count: 0,
filtered: undefined
});
});
});
describe('Leetspeak Detection', () => {
test('should detect leetspeak profanity', () => {
const testCases = [
'This is sh1t',
'F@ck this',
'What the f*ck',
'This is bull$hit',
'D@mn it',
'A$$hole behavior'
];
testCases.forEach(text => {
const result = filter.analyzeProfanity(text);
expect(result.hasProfanity).toBe(true);
expect(result.count).toBeGreaterThan(0);
});
});
test('should detect spaced out words', () => {
const result = filter.analyzeProfanity('f u c k this');
expect(result.hasProfanity).toBe(true);
expect(result.count).toBeGreaterThan(0);
});
});
describe('Severity Levels', () => {
test('should classify high severity words correctly', () => {
const highSeverityWords = ['kill', 'murder', 'terrorist', 'rape'];
highSeverityWords.forEach(word => {
const result = filter.analyzeProfanity(`This is ${word}`);
expect(result.hasProfanity).toBe(true);
expect(result.severity).toBe('high');
});
});
test('should classify medium severity words correctly', () => {
const mediumSeverityWords = ['fuck', 'shit', 'bitch'];
mediumSeverityWords.forEach(word => {
const result = filter.analyzeProfanity(`This is ${word}`);
expect(result.hasProfanity).toBe(true);
expect(result.severity).toBe('medium');
});
});
test('should classify low severity words correctly', () => {
const lowSeverityWords = ['damn', 'hell', 'crap'];
lowSeverityWords.forEach(word => {
const result = filter.analyzeProfanity(`This is ${word}`);
expect(result.hasProfanity).toBe(true);
expect(result.severity).toBe('low');
});
});
test('should use highest severity when multiple words present', () => {
const result = filter.analyzeProfanity('damn this fucking terrorist');
expect(result.hasProfanity).toBe(true);
expect(result.severity).toBe('high'); // terrorist is high severity
});
});
describe('Text Filtering', () => {
test('should filter profanity with asterisks', () => {
const result = filter.analyzeProfanity('This is fucking shit');
expect(result.filtered).toContain('***');
expect(result.filtered).not.toContain('fuck');
expect(result.filtered).not.toContain('shit');
});
test('should preserve clean parts of text', () => {
const result = filter.analyzeProfanity('This damn road is slippery');
expect(result.filtered).toContain('road is slippery');
expect(result.filtered).toContain('***');
expect(result.filtered).not.toContain('damn');
});
});
describe('Custom Words Management', () => {
test('should add custom profanity word', async () => {
const result = await filter.addCustomWord('testword', 'medium', 'custom', 'admin');
expect(result.word).toBe('testword');
expect(result.severity).toBe('medium');
expect(result.category).toBe('custom');
});
test('should prevent duplicate custom words', async () => {
await filter.addCustomWord('testword', 'medium', 'custom', 'admin');
await expect(
filter.addCustomWord('testword', 'high', 'custom', 'admin')
).rejects.toThrow('Word already exists in the filter');
});
test('should detect custom words after reload', async () => {
await filter.addCustomWord('customfoulword', 'high', 'custom', 'admin');
await filter.loadCustomWords();
const result = filter.analyzeProfanity('This is customfoulword');
expect(result.hasProfanity).toBe(true);
expect(result.count).toBe(1);
expect(result.severity).toBe('high');
});
test('should get all custom words', async () => {
await filter.addCustomWord('word1', 'low', 'custom', 'admin');
await filter.addCustomWord('word2', 'high', 'custom', 'admin');
const words = await filter.getCustomWords();
expect(words).toHaveLength(2);
expect(words.map(w => w.word)).toContain('word1');
expect(words.map(w => w.word)).toContain('word2');
});
test('should update custom word', async () => {
const added = await filter.addCustomWord('updateword', 'low', 'custom', 'admin');
const result = await filter.updateCustomWord(added.id, {
word: 'updatedword',
severity: 'high',
category: 'updated'
});
expect(result.updated).toBe(true);
expect(result.changes).toBe(1);
});
test('should remove custom word', async () => {
const added = await filter.addCustomWord('removeword', 'medium', 'custom', 'admin');
const result = await filter.removeCustomWord(added.id);
expect(result.deleted).toBe(true);
expect(result.changes).toBe(1);
});
test('should handle removing non-existent word', async () => {
await expect(
filter.removeCustomWord(99999)
).rejects.toThrow('Word not found');
});
});
describe('Edge Cases', () => {
test('should handle very long text', () => {
const longText = 'This road is slippery '.repeat(100) + 'shit';
const result = filter.analyzeProfanity(longText);
expect(result.hasProfanity).toBe(true);
expect(result.count).toBe(1);
});
test('should handle text with only profanity', () => {
const result = filter.analyzeProfanity('fuck');
expect(result.hasProfanity).toBe(true);
expect(result.count).toBe(1);
expect(result.filtered).toBe('****');
});
test('should handle mixed case profanity', () => {
const result = filter.analyzeProfanity('This FUCKING road is SHIT');
expect(result.hasProfanity).toBe(true);
expect(result.count).toBe(2);
});
test('should handle profanity with punctuation', () => {
const result = filter.analyzeProfanity('Fuck! This shit, damn...');
expect(result.hasProfanity).toBe(true);
expect(result.count).toBe(3);
});
test('should not detect profanity in legitimate words containing profane substrings', () => {
const legitimateWords = [
'assessment', // contains 'ass'
'classic', // contains 'ass'
'assistance', // contains 'ass'
'cassette' // contains 'ass'
];
legitimateWords.forEach(word => {
const result = filter.analyzeProfanity(`This is a ${word}`);
expect(result.hasProfanity).toBe(false);
});
});
});
describe('Real-world Road Condition Examples', () => {
test('should allow legitimate road condition descriptions', () => {
const legitimateDescriptions = [
'Multiple vehicles stuck due to black ice',
'Road very slippery, saw 3 accidents this morning',
'Ice forming on bridges, drive carefully',
'Heavy snow, visibility poor',
'Salt trucks active, conditions improving',
'Watched 2 cars slide into ditch',
'School buses delayed due to ice',
'Emergency vehicles on scene',
'Road closed between Main and Oak'
];
legitimateDescriptions.forEach(description => {
const result = filter.analyzeProfanity(description);
expect(result.hasProfanity).toBe(false);
});
});
test('should reject inappropriate descriptions', () => {
const inappropriateDescriptions = [
'This fucking road is terrible',
'Shit everywhere, can\'t drive',
'Damn ice caused accident',
'These asshole drivers are crazy',
'What the hell is wrong with road crews'
];
inappropriateDescriptions.forEach(description => {
const result = filter.analyzeProfanity(description);
expect(result.hasProfanity).toBe(true);
expect(result.count).toBeGreaterThan(0);
});
});
});
describe('Performance', () => {
test('should handle multiple rapid analyses', () => {
const startTime = Date.now();
for (let i = 0; i < 100; i++) {
filter.analyzeProfanity('This is a test message with some words');
}
const endTime = Date.now();
const duration = endTime - startTime;
// Should complete 100 analyses in under 1 second
expect(duration).toBeLessThan(1000);
});
});
});

411
tests/routes.test.js Normal file
View file

@ -0,0 +1,411 @@
const request = require('supertest');
const express = require('express');
const sqlite3 = require('sqlite3').verbose();
const ProfanityFilter = require('../profanity-filter');
const locationRoutes = require('../routes/locations');
const adminRoutes = require('../routes/admin');
const configRoutes = require('../routes/config');
describe('API Routes', () => {
let app;
let db;
let profanityFilter;
beforeEach(async () => {
// Create test app and database
app = express();
app.use(express.json());
db = new sqlite3.Database(':memory:');
// Create profanity filter with test database path
profanityFilter = new ProfanityFilter(':memory:');
// Wait for profanity filter initialization
await new Promise(resolve => setTimeout(resolve, 300));
// Initialize locations table synchronously
await new Promise((resolve, reject) => {
db.serialize(() => {
db.run(`CREATE TABLE IF NOT EXISTS locations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
address TEXT NOT NULL,
latitude REAL,
longitude REAL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
description TEXT,
persistent INTEGER DEFAULT 0,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)`, (err) => {
if (err) {
console.error('Error creating locations table:', err);
reject(err);
} else {
resolve();
}
});
});
});
// Reload profanity filter patterns after database is ready
await profanityFilter.loadCustomWords();
// Setup routes
app.use('/api/locations', locationRoutes(db, profanityFilter));
app.use('/api/config', configRoutes());
// Mock admin authentication for testing
const mockAuth = (req, res, next) => next();
app.use('/api/admin', adminRoutes(db, profanityFilter, mockAuth));
});
afterEach(async () => {
// Wait longer for all pending operations to complete
await new Promise(resolve => setTimeout(resolve, 200));
// Close profanity filter database first
if (profanityFilter) {
profanityFilter.close();
}
// Wait a bit more
await new Promise(resolve => setTimeout(resolve, 100));
// Close main test database
if (db) {
await new Promise(resolve => {
db.close((err) => {
if (err) {
console.error('Error closing database:', err);
}
resolve();
});
});
}
});
describe('Location Routes', () => {
describe('POST /api/locations', () => {
test('should accept clean location submission', async () => {
const response = await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681,
description: 'Road is very slippery with black ice'
});
expect(response.status).toBe(200);
expect(response.body.address).toBe('123 Main St, Grand Rapids, MI');
expect(response.body.description).toBe('Road is very slippery with black ice');
expect(response.body.id).toBeDefined();
});
test('should reject submission with profanity', async () => {
const response = await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681,
description: 'This fucking road is terrible'
});
expect(response.status).toBe(400);
expect(response.body.error).toBe('Submission rejected');
expect(response.body.message).toContain('inappropriate language');
expect(response.body.details.wordCount).toBeGreaterThan(0);
});
test('should reject submission with single profanity word', async () => {
const response = await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681,
description: 'Road is shit'
});
expect(response.status).toBe(400);
expect(response.body.error).toBe('Submission rejected');
expect(response.body.details.wordCount).toBe(1);
});
test('should reject submission with leetspeak profanity', async () => {
const response = await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681,
description: 'Road is sh1t'
});
expect(response.status).toBe(400);
expect(response.body.error).toBe('Submission rejected');
});
test('should require address field', async () => {
const response = await request(app)
.post('/api/locations')
.send({
latitude: 42.9634,
longitude: -85.6681,
description: 'Road is slippery'
});
expect(response.status).toBe(400);
expect(response.body.error).toBe('Address is required');
});
test('should accept submission without description', async () => {
const response = await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681
});
expect(response.status).toBe(200);
expect(response.body.address).toBe('123 Main St, Grand Rapids, MI');
});
test('should handle empty description', async () => {
const response = await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681,
description: ''
});
expect(response.status).toBe(200);
});
});
describe('GET /api/locations', () => {
test('should return empty array when no locations', async () => {
const response = await request(app)
.get('/api/locations');
expect(response.status).toBe(200);
expect(response.body).toEqual([]);
});
test('should return locations after adding them', async () => {
// Add a location first
await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681,
description: 'Road is slippery'
});
const response = await request(app)
.get('/api/locations');
expect(response.status).toBe(200);
expect(response.body).toHaveLength(1);
expect(response.body[0].address).toBe('123 Main St, Grand Rapids, MI');
});
});
});
describe('Config Routes', () => {
describe('GET /api/config', () => {
test('should return config with mapbox settings', async () => {
const response = await request(app)
.get('/api/config');
expect(response.status).toBe(200);
expect(response.body).toHaveProperty('hasMapbox');
expect(response.body).toHaveProperty('mapboxAccessToken');
});
});
});
describe('Admin Routes', () => {
describe('Profanity Management', () => {
test('should get empty custom words list initially', async () => {
const response = await request(app)
.get('/api/admin/profanity-words');
expect(response.status).toBe(200);
expect(response.body).toEqual([]);
});
test('should add custom profanity word', async () => {
const response = await request(app)
.post('/api/admin/profanity-words')
.send({
word: 'custombaword',
severity: 'high',
category: 'test'
});
expect(response.status).toBe(200);
expect(response.body.word).toBe('custombaword');
expect(response.body.severity).toBe('high');
});
test('should reject duplicate custom words', async () => {
// Add first word
await request(app)
.post('/api/admin/profanity-words')
.send({
word: 'duplicate',
severity: 'medium'
});
// Try to add same word again
const response = await request(app)
.post('/api/admin/profanity-words')
.send({
word: 'duplicate',
severity: 'high'
});
expect(response.status).toBe(409);
expect(response.body.error).toContain('already exists');
});
test('should validate custom word input', async () => {
const invalidInputs = [
{ word: '', severity: 'medium' },
{ word: ' ', severity: 'medium' },
{ word: 'test', severity: 'invalid' },
{ severity: 'medium' }
];
for (const input of invalidInputs) {
const response = await request(app)
.post('/api/admin/profanity-words')
.send(input);
expect(response.status).toBe(400);
}
});
test('should test profanity filter', async () => {
const response = await request(app)
.post('/api/admin/test-profanity')
.send({
text: 'This fucking road is terrible'
});
expect(response.status).toBe(200);
expect(response.body.original).toBe('This fucking road is terrible');
expect(response.body.analysis.hasProfanity).toBe(true);
expect(response.body.filtered).toContain('***');
});
});
describe('Location Management', () => {
test('should get all locations for admin', async () => {
// Add a location first
await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681,
description: 'Road is slippery'
});
const response = await request(app)
.get('/api/admin/locations');
expect(response.status).toBe(200);
expect(response.body).toHaveLength(1);
expect(response.body[0]).toHaveProperty('isActive');
});
});
});
describe('Error Handling', () => {
test('should handle malformed JSON', async () => {
const response = await request(app)
.post('/api/locations')
.set('Content-Type', 'application/json')
.send('{ invalid json }');
expect(response.status).toBe(400);
});
test('should handle missing content-type', async () => {
const response = await request(app)
.post('/api/locations')
.send('address=test');
// Should still work as express.json() is flexible
expect(response.status).toBe(400); // Will fail validation for missing required fields
});
});
describe('Integration Tests', () => {
test('should handle complete workflow with profanity', async () => {
// 1. Try to submit with profanity (should be rejected)
const rejectedResponse = await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681,
description: 'This damn road is fucking terrible'
});
expect(rejectedResponse.status).toBe(400);
expect(rejectedResponse.body.error).toBe('Submission rejected');
// 2. Submit clean version (should be accepted)
const acceptedResponse = await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681,
description: 'Road is very slippery and dangerous'
});
expect(acceptedResponse.status).toBe(200);
// 3. Verify it appears in the list
const listResponse = await request(app)
.get('/api/locations');
expect(listResponse.status).toBe(200);
expect(listResponse.body).toHaveLength(1);
expect(listResponse.body[0].description).toBe('Road is very slippery and dangerous');
});
test('should handle custom profanity words in submissions', async () => {
// 1. Add custom profanity word
await request(app)
.post('/api/admin/profanity-words')
.send({
word: 'customoffensive',
severity: 'high',
category: 'test'
});
// 2. Try to submit with custom word (should be rejected)
const response = await request(app)
.post('/api/locations')
.send({
address: '123 Main St, Grand Rapids, MI',
latitude: 42.9634,
longitude: -85.6681,
description: 'This road is customoffensive'
});
expect(response.status).toBe(400);
expect(response.body.error).toBe('Submission rejected');
});
});
});

16
tests/setup.js Normal file
View file

@ -0,0 +1,16 @@
// Test setup file for Jest
// This file runs before each test suite
// Suppress console.log during tests unless debugging
if (!process.env.DEBUG_TESTS) {
console.log = jest.fn();
console.warn = jest.fn();
console.error = jest.fn();
}
// Set test environment variables
process.env.NODE_ENV = 'test';
process.env.ADMIN_PASSWORD = 'test_admin_password';
// Global test timeout
jest.setTimeout(30000);