def preprocess_word(word):

word = word.strip(’\’"?!,.():;')

word = re.sub(r’(.)\1+’, r’\1\1’, word)

word = re.sub(r’(-|\’)’, ‘’, word) return word

def is_valid_word(word):

return (re.search(r’^[a-zA-Z][a-z0-9A-Z\._]*$’, word) is not None)

def handle_emojis(tweet):

tweet = re.sub(r’(:\s?\)|:-\)|\(\s?:|\(-:|:'\))’, ’ EMO_POS ‘, tweet)

tweet = re.sub(r’(:\s?D|:-D|x-?D|X-?D)’, ’ EMO_POS ‘, tweet)

tweet = re.sub(r’(<3|:\*)’, ’ EMO_POS ‘, tweet)

tweet = re.sub(r’(;-?\)|;-?D|\(-?;)’, ’ EMO_POS ‘, tweet)

tweet = re.sub(r’(:\s?\(|:-\(|\)\s?:|\)-:)’, ’ EMO_NEG ‘, tweet)

tweet = re.sub(r’(:,\(|:\’\(|:"\()’, ’ EMO_NEG ‘, tweet) return tweet

def preprocess_tweet(tweet): processed_tweet = []

tweet = tweet.lower()

tweet = re.sub(r’((www\.[§]+)|(https?://[§]+))’, ’ URL ‘, tweet)

tweet = re.sub(r’@[§]+’, ‘USER_MENTION’, tweet)

tweet = re.sub(r’#(§+)’, r’ \1 ‘, tweet)

tweet = re.sub(r’\brt\b’, ‘’, tweet)

tweet = re.sub(r’\.{2,}’, ’ ‘, tweet)

tweet = tweet.strip(’ “\’’)

tweet = handle_emojis(tweet)

tweet = re.sub(r’\s+’, ’ ‘, tweet) words = tweet.split()

for word in words: word = preprocess_word(word) if is_valid_word(word): if use_stemmer: word = str(porter_stemmer.stem(word)) processed_tweet.append(word)

return ’ ‘.join(processed_tweet)

def preprocess_csv(csv_file_name, processed_file_name, test_file=False): save_to_file = open(processed_file_name, ‘w’)

with open(csv_file_name, ‘r’) as csv: lines = csv.readlines() total = len(lines) for i, line in enumerate(lines): tweet_id = line[:line.find(’,’)] if not test_file: line = line[1 + line.find(’,’):] positive = int(line[:line.find(’,’)]) line = line[1 + line.find(’,’):] tweet = line processed_tweet = preprocess_tweet(tweet) if not test_file: save_to_file.write(’%s,%d,%s\n’ % (tweet_id, positive, processed_tweet)) else: save_to_file.write(’%s,%s\n’ % (tweet_id, processed_tweet)) write_status(i + 1, total) save_to_file.close() print ‘\nSaved processed tweets to: %s’ % processed_file_name return processed_file_name

if name = '__main__': if len(sys.argv) ! 2: print ‘Usage: python preprocess.py ’ exit() use_stemmer = False csv_file_name = sys.argv[1] processed_file_name = sys.argv[1][:-4] + ‘-processed.csv’ if use_stemmer: porter_stemmer = PorterStemmer() processed_file_name = sys.argv[1][:-4] + ‘-processed-stemmed.csv’ preprocess_csv(csv_file_name, processed_file_name, test_file=False)

(#[a-z]+)|(#[a-z]+[A-Z]*)

#([a-z]+)([A-Z][^A-Z]*)*[.]?