From 9306cb60c32082c5403931de0823a9fd5daa196c Mon Sep 17 00:00:00 2001
From: Jason Katz-Brown <jason@airbnb.com>
Date: Sun, 25 Aug 2013 02:17:13 -0700
Subject: Initial git commit.

---
 .../perl_tools/generate_raw_words.pl               | 23 ++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100755 new_language_generation/perl_tools/generate_raw_words.pl

(limited to 'new_language_generation/perl_tools/generate_raw_words.pl')

diff --git a/new_language_generation/perl_tools/generate_raw_words.pl b/new_language_generation/perl_tools/generate_raw_words.pl
new file mode 100755
index 0000000..b32d810
--- /dev/null
+++ b/new_language_generation/perl_tools/generate_raw_words.pl
@@ -0,0 +1,23 @@
+#!/usr/bin/perl
+
+# USAGE: generate_raw_words.pl generate_words_output
+
+binmode STDOUT, ':utf8';
+
+sub read_and_spit {
+	for my $arg (@ARGV) {
+		open (my $input, "<:encoding(utf8)", $arg);
+
+		while (<$input>) {
+			chomp;
+			my ($word, $count) = split(/\s/, $_);
+
+			my $clean_word = $word;
+			$clean_word =~ s/;//g;
+
+			print "$clean_word\n";
+		}
+	}
+}
+
+read_and_spit();
-- 
cgit v1.2.3