Mega Code Archive
Filter text with a stop word list
<![CDATA[
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define MAXTOKENS 256
#define MAXLINE 1024
#define MINLEN 3
#define STMINLEN 2
struct tnode {
char *word;
int count;
struct tnode *left, *right;
};
struct tnode *buildstoptree(char *, struct tnode *);
struct tnode *addtree(struct tnode *, char *);
struct tnode *findstopword(struct tnode *, char *);
struct tnode *talloc(void);
void freetree(struct tnode *);
char **split(char *, char *);
int main(int argc, char *argv[]) {
/* delim does not include \' [\047] quote */
char *delim = ".,:;`\"+-_(){}[]<>*&^%$#@!?~/|\\= \t\r\n1234567890";
char **tokens = NULL;
struct tnode *root = {0};
struct tnode *querry = {0};
char line[MAXLINE];
int i = 0;
if(argc != 2) {
fprintf(stderr, "Usage: tokstop STOPLIST.txt\n");
return 1;
}
root = buildstoptree(argv[1], root);
if(root == NULL)
return 1;
while(fgets(line, MAXLINE, stdin) != NULL) {
if(strlen(line) < MINLEN)
continue;
tokens = split(line, delim);
for(i = 0; tokens[i] != NULL; i++) {
querry = findstopword(root, tokens[i]);
if(querry == NULL)
printf("%s ", tokens[i]);
}
for(i = 0; tokens[i] != NULL; i++)
free(tokens[i]);
free(tokens[i]);
printf("\n");
}
freetree(root);
return 0;
}
/* read stoplist into binary tree, expects one entry per line */
struct tnode *buildstoptree(char *fname, struct tnode *p) {
FILE *fp = {0};
char line[MAXLINE];
int len = 0, lcount = 0;
fp = fopen(fname, "r");
if(fp == NULL) {
fprintf(stderr, "Error - fopen(%s)\n", fname);
return NULL;
}
while(fgets(line, MAXLINE, fp) != NULL) {
len = strlen(line);
if(len < STMINLEN)
continue;
else
lcount++;
if(line[len - 1] == '\n')
line[--len] = '\0';
p = addtree(p, line);
}
if(lcount == 0) {
fprintf(stderr, "Error - Zero stopwords..\n");
return NULL;
}
fclose(fp);
return p;
}
/* split string into tokens, return token array */
char **split(char *string, char *delim) {
char **tokens = NULL;
char *working = NULL;
char *token = NULL;
int idx = 0;
tokens = malloc(sizeof(char *) * MAXTOKENS);
if(tokens == NULL)
return NULL;
working = malloc(sizeof(char) * strlen(string) + 1);
if(working == NULL)
return NULL;
/* to make sure, copy string to a safe place */
strcpy(working, string);
for(idx = 0; idx < MAXTOKENS; idx++)
tokens[idx] = NULL;
token = strtok(working, delim);
idx = 0;
/* always keep the last entry NULL terminated */
while((idx < (MAXTOKENS - 1)) && (token != NULL)) {
tokens[idx] = malloc(sizeof(char) * strlen(token) + 1);
if(tokens[idx] != NULL) {
strcpy(tokens[idx], token);
idx++;
token = strtok(NULL, delim);
}
}
free(working);
return tokens;
}
/* install word in binary tree */
struct tnode *addtree(struct tnode *p, char *w) {
int cond;
if(p == NULL) {
p = talloc();
p->word = strdup(w);
p->count = 1;
p->left = p->right = NULL;
} else if((cond = strcmp(w, p->word)) == 0)
p->count++;
else if(cond < 0)
p->left = addtree(p->left, w);
else
p->right = addtree(p->right, w);
return p;
}
/* make new tnode */
struct tnode *talloc(void) {
return(struct tnode *)malloc(sizeof(struct tnode));
}
/* find value w in binary tree */
struct tnode *findstopword(struct tnode *p, char *w) {
struct tnode *temp;
int cond = 0;
temp = p;
while(temp != NULL) {
if((cond = strcmp(temp->word, w)) == 0)
return temp;
else if(cond > 0)
temp = temp->left;
else
temp = temp->right;
}
return NULL;
}
/* free binary tree */
void freetree(struct tnode *p) {
if(p != NULL) {
free(p->left);
free(p->right);
free(p->word);
free(p);
}
}
]]>