diff --git a/2018-komp-ling/practicals/Unigram part-of-speech tagger/Unigram_part_of_speech_tagger_response.md b/2018-komp-ling/practicals/Unigram part-of-speech tagger/Unigram_part_of_speech_tagger_response.md new file mode 100644 index 00000000..b370d769 --- /dev/null +++ b/2018-komp-ling/practicals/Unigram part-of-speech tagger/Unigram_part_of_speech_tagger_response.md @@ -0,0 +1,68 @@ +# matplotlib + +Получение рангов для нашего текста + +```` +import matplotlib.pyplot as plt + +freq = [] +ranks = [] + +#load data +with open('./../freq.txt', 'r') as f: + f = f.readlines() +for line in f: + line = line.strip('\n') + (f, w) = line.split('\t') + freq.append((int(f), w)) + +freq.sort(reverse=True) + +#ranking data +rank = 1 +min = freq[0][0] +for i in range(0, len(freq)): + if freq[i][0] < min: + rank +=1 + min = freq[i][0] + ranks.append([rank, freq[i][0], freq[i][1]]) + +#do the plots +x = [] +y = [] +for line in ranks: + row = line + x.append(int(row[0])) + y.append(int(row[1])) +plt.plot(x, y, 'b*') +plt.show() +```` +# ElementTree + +### How would you get just the Icelandic line and the gloss line ? + +```` +for tier in root.findall('.//tier'): + if tier.attrib['id'] == 'n': + for item in tier.findall('.//item'): + if item.attrib['tag'] != 'T': # here is the condition + print(item.text) +```` + +# scikit learn + +### Perceptron answers +```` +- #хоругвь# incorrect class: 0 correct class: 1 +- #обувь# incorrect class: 0 correct class: 1 +- #морковь# incorrect class: 0 correct class: 1 +- #бровь# incorrect class: 0 correct class: 1 +- #церковь# incorrect class: 0 correct class: 1 +0.982857142857142856 +```` +To improve the quiality of our model we should use MLP, or deeper (than 1 layer) models + +# Screenscraping + +done in __screencap.py__ + diff --git a/2018-komp-ling/practicals/Unigram part-of-speech tagger/screencap.py b/2018-komp-ling/practicals/Unigram part-of-speech tagger/screencap.py new file mode 100644 index 00000000..9421d8c5 --- /dev/null +++ b/2018-komp-ling/practicals/Unigram part-of-speech tagger/screencap.py @@ -0,0 +1,43 @@ +#дерево +import sys + +def strip_html(h): + output = '' + inTag = False + for c in h: + if c == '<': + inTag = True + continue + if c == '>': + inTag = False + continue + if not inTag: + output += c + return output + +stem = '_' +zkod = '_' +ipa = '_' + + +h1 = '_' +for line in sys.stdin.readlines(): + line = line.strip() + text = strip_html(line) + if line.count('

') > 0: + h1 = strip_html(line) + if h1 != 'Русский': + continue + if text.count('Корень:') > 0: + stem = text.split(':')[1].split(';')[0] + if text.count('МФА') > 0: + ipa = text.split(';')[3].split('&')[0] + if text.count('тип склонения') > 0: + zkod = text.split('тип склонения')[1].strip().split(' ')[0].strip("^") + + +if stem != '_' and zkod != '_' and ipa != '_': + print('%s\t%s\t%s' % (stem, zkod, ipa)) + stem = '_' + zkod = '_' + ipa = '_' diff --git a/2018-komp-ling/practicals/Unigram-part-of-speech-tagger-response.md b/2018-komp-ling/practicals/Unigram-part-of-speech-tagger-response.md new file mode 100644 index 00000000..b370d769 --- /dev/null +++ b/2018-komp-ling/practicals/Unigram-part-of-speech-tagger-response.md @@ -0,0 +1,68 @@ +# matplotlib + +Получение рангов для нашего текста + +```` +import matplotlib.pyplot as plt + +freq = [] +ranks = [] + +#load data +with open('./../freq.txt', 'r') as f: + f = f.readlines() +for line in f: + line = line.strip('\n') + (f, w) = line.split('\t') + freq.append((int(f), w)) + +freq.sort(reverse=True) + +#ranking data +rank = 1 +min = freq[0][0] +for i in range(0, len(freq)): + if freq[i][0] < min: + rank +=1 + min = freq[i][0] + ranks.append([rank, freq[i][0], freq[i][1]]) + +#do the plots +x = [] +y = [] +for line in ranks: + row = line + x.append(int(row[0])) + y.append(int(row[1])) +plt.plot(x, y, 'b*') +plt.show() +```` +# ElementTree + +### How would you get just the Icelandic line and the gloss line ? + +```` +for tier in root.findall('.//tier'): + if tier.attrib['id'] == 'n': + for item in tier.findall('.//item'): + if item.attrib['tag'] != 'T': # here is the condition + print(item.text) +```` + +# scikit learn + +### Perceptron answers +```` +- #хоругвь# incorrect class: 0 correct class: 1 +- #обувь# incorrect class: 0 correct class: 1 +- #морковь# incorrect class: 0 correct class: 1 +- #бровь# incorrect class: 0 correct class: 1 +- #церковь# incorrect class: 0 correct class: 1 +0.982857142857142856 +```` +To improve the quiality of our model we should use MLP, or deeper (than 1 layer) models + +# Screenscraping + +done in __screencap.py__ + diff --git a/2018-komp-ling/practicals/segmentation-response.md b/2018-komp-ling/practicals/segmentation-response.md new file mode 100644 index 00000000..fa2dc210 --- /dev/null +++ b/2018-komp-ling/practicals/segmentation-response.md @@ -0,0 +1,19 @@ + + +
+ +

Обзор двух библиотек по токенизированию предложений

+ +В данном отчете было использованы две библиотеки по токенеизорванию спредложений из текста: pragmatic segmenter (Ruby) и NLTK (Python). В качестве тестового текста был использован кусок дампа русской Википедии. +

Pragmatic segmenter (Ruby)

+Pragmatic segmenter - это бибилотека для Ruby, основанная на правилах. При парсинге русской википедии данная библиотека показала качество ниже среднего. Большинство сокращений, инициалы имен и т.д. неправильно были разделены на предложения. +В общем библиотека больше расчитана на языки латинского алфавита. + +

NLTK (Python)

+ +sent_tokenize() - это функция библиотеки NLTK по определению границ предложения. Но на самом деле это алгоритм машинного обучения без учителя, который можно обучить самомстоятельно. В бибилотеке NLTK уже есть набор pre-trained моделей, в том числе и для русского языка. В общем данная библиотека показала себя лучше, чем Ruby. Большинство сокращений и инициалов выделены правильно, единтсвенную проблему составляет сокращения с пробелами внутри. + +
diff --git a/2018-komp-ling/practicals/segmentation/segmentation-response.md b/2018-komp-ling/practicals/segmentation/segmentation-response.md new file mode 100644 index 00000000..fa2dc210 --- /dev/null +++ b/2018-komp-ling/practicals/segmentation/segmentation-response.md @@ -0,0 +1,19 @@ + + +
+ +

Обзор двух библиотек по токенизированию предложений

+ +В данном отчете было использованы две библиотеки по токенеизорванию спредложений из текста: pragmatic segmenter (Ruby) и NLTK (Python). В качестве тестового текста был использован кусок дампа русской Википедии. +

Pragmatic segmenter (Ruby)

+Pragmatic segmenter - это бибилотека для Ruby, основанная на правилах. При парсинге русской википедии данная библиотека показала качество ниже среднего. Большинство сокращений, инициалы имен и т.д. неправильно были разделены на предложения. +В общем библиотека больше расчитана на языки латинского алфавита. + +

NLTK (Python)

+ +sent_tokenize() - это функция библиотеки NLTK по определению границ предложения. Но на самом деле это алгоритм машинного обучения без учителя, который можно обучить самомстоятельно. В бибилотеке NLTK уже есть набор pre-trained моделей, в том числе и для русского языка. В общем данная библиотека показала себя лучше, чем Ruby. Большинство сокращений и инициалов выделены правильно, единтсвенную проблему составляет сокращения с пробелами внутри. + +
diff --git a/2018-komp-ling/practicals/transliteration-response.md b/2018-komp-ling/practicals/transliteration-response.md new file mode 100644 index 00000000..6532626d --- /dev/null +++ b/2018-komp-ling/practicals/transliteration-response.md @@ -0,0 +1,42 @@ +# Practical 2: Transliteration (engineering) + +
+ +## Questions +What to do with ambiguous letters ? For example, Cyrillic `е' could be either je or e. + +Can you think of a way that you could provide mappings from many characters to one character ? + For example sh → ш or дж → c ? +How might you make different mapping rules for characters at the beginning or end of the string ? + + +### Правила для транслитерации + +Основная идея - это начинать транслитерацию со сложных, многобуквенных преобразований (ч - tch). Например: +>Шарик -- sh-арик -- sharik + +Далее нужно заменить все гласные в начале и в конце слова (Я - ya). +>яблоко -- ya-блоко -- yabloko + +После чего уже можно переходить на простые однобуквенные преобразования (у - u) +>мед -- med + +## Методы +### Кодировка-декодировка с помощью KOI-8R + +Транслитерация с помощью кодировки KOI-8R - не самый эффективный метод транслитерации текста. +Но он обеспечивает некоторые особенности, которые не доступны другим способам: + + 1)Возможность восстановить первоначальный текст + + 2)Правила кодировки уже заданы + +Метод транслитерации с помощью KOI-8R представлен в файле transliterate_koi8r.py + +### Кодировка-декодировка с помощью правил + +Правила для транслетерации находятся в файле rules.txt. + В нем заданы правила для согланых, гласных, а также для гласных в начале слова. + + +
diff --git a/2018-komp-ling/practicals/transliteration/rank.py b/2018-komp-ling/practicals/transliteration/rank.py new file mode 100644 index 00000000..476ab50c --- /dev/null +++ b/2018-komp-ling/practicals/transliteration/rank.py @@ -0,0 +1,44 @@ +#!/usr/bin/python + +import sys, getopt + +def main(argv): + inputfile = '' + outputfile = 'ranked.txt' + try: + opts, args = getopt.getopt(argv,"hi:o:",["ifile=","ofile="]) + except getopt.GetoptError: + print('test.py -i -o ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print( 'test.py -i -o ') + sys.exit() + elif opt in ("-i", "--ifile"): + inputfile = arg + elif opt in ("-o", "--ofile"): + outputfile = arg + print( 'Input file is:', inputfile) + print( 'Output file is:', outputfile) + + freq = [] + with open(inputfile, 'r',encoding='utf8') as fd: + for line in fd.readlines(): + line = line.strip('\n') + (f, w) = line.split('\t') + freq.append((int(f), w)) + rank = 1 + min = freq[0][0] + ranks = [] + for i in range(0, len(freq)): + if freq[i][0] < min: + rank = rank + 1 + min = freq[i][0] + ranks.append((rank, freq[i][0], freq[i][1])) + + with open(outputfile, 'w+',encoding='utf8') as fd: + for w in vocab: + fd.write(ranks) + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/2018-komp-ling/practicals/transliteration/rules.txt b/2018-komp-ling/practicals/transliteration/rules.txt new file mode 100644 index 00000000..a3bdc243 --- /dev/null +++ b/2018-komp-ling/practicals/transliteration/rules.txt @@ -0,0 +1,36 @@ + я ya + ю yu + е ye +а a +б b +в v +г g +д d +е e +ё yo +ж zsh +з z +и i +й y +к k +л l +м m +н n +о o +п p +р r +с s +т t +у u +ф f +х h +ц ts +ч tch +ш ch +щ scsh +ъ ' +ы uy +ь ' +э a +ю u +я a diff --git a/2018-komp-ling/practicals/transliteration/transliterate.py b/2018-komp-ling/practicals/transliteration/transliterate.py new file mode 100644 index 00000000..73974730 --- /dev/null +++ b/2018-komp-ling/practicals/transliteration/transliterate.py @@ -0,0 +1,58 @@ +#!/usr/bin/python + +import sys, getopt + +def main(argv): + inputfile = '' + outputfile = '__translitareted.conllu' + try: + opts, args = getopt.getopt(argv,"hi:o:",["ifile=","ofile="]) + except getopt.GetoptError: + print('test.py -i -o ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print( 'test.py -i -o ') + sys.exit() + elif opt in ("-i", "--ifile"): + inputfile = arg + elif opt in ("-o", "--ofile"): + outputfile = arg + print( 'Input file is:', inputfile) + print( 'Output file is:', outputfile) + + vocab = [] + test = open(inputfile,'r',encoding='utf8') + for line in test.readlines(): + if '\t' not in line: + continue + row = line.replace('\n','').split('\t') + if len(row) != 10: + continue + vocab.append(row) + test.close() + + for i in enumerate(vocab): + try: + bar = i[1][1] + for i in enumerate(bar): + if i[0] == 0: + try: + bar = bar.replace(i[1],rules_for[' '+i[1]]) + except: + bar = bar.replace(i[1],rules_for[i[1]]) + else: + try: + bar = bar.replace(i[1],rules_for[i[1]]) + except: + pass + vocab[i[0]][9] = bar + except: + continue + + with open(outputfile, 'w+',encoding='utf8') as fd: + for w in vocab: + fd.write('\t'.join(w)+'\n') + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/2018-komp-ling/practicals/transliteration/transliterate_koi8r.py b/2018-komp-ling/practicals/transliteration/transliterate_koi8r.py new file mode 100644 index 00000000..fe6799f1 --- /dev/null +++ b/2018-komp-ling/practicals/transliteration/transliterate_koi8r.py @@ -0,0 +1,49 @@ +#!/usr/bin/python + +import sys, getopt + +def main(argv): + inputfile = '' + outputfile = '__translitareted.conllu' + try: + opts, args = getopt.getopt(argv,"hi:o:",["ifile=","ofile="]) + except getopt.GetoptError: + print('test.py -i -o ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print( 'test.py -i -o ') + sys.exit() + elif opt in ("-i", "--ifile"): + inputfile = arg + elif opt in ("-o", "--ofile"): + outputfile = arg + print( 'Input file is:', inputfile) + print( 'Output file is:', outputfile) + + vocab = [] + test = open(inputfile,'r',encoding='utf8') + for line in test.readlines(): + if '\t' not in line: + continue + row = line.replace('\n','').split('\t') + if len(row) != 10: + continue + vocab.append(row) + test.close() + + #magic KOI-8r + for i in enumerate(vocab): + try: + oldone = i[1][1].encode('koi8-r') + newone = ''.join([chr(c & 0x7F) for c in oldone]) + vocab[i[0]][9] = newone + except: + continue + + with open(outputfile, 'w+',encoding='utf8') as fd: + for w in vocab: + fd.write('\t'.join(w)+'\n') + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/2018-komp-ling/practicals/transliteration/transliteration-response.md b/2018-komp-ling/practicals/transliteration/transliteration-response.md new file mode 100644 index 00000000..6532626d --- /dev/null +++ b/2018-komp-ling/practicals/transliteration/transliteration-response.md @@ -0,0 +1,42 @@ +# Practical 2: Transliteration (engineering) + +
+ +## Questions +What to do with ambiguous letters ? For example, Cyrillic `е' could be either je or e. + +Can you think of a way that you could provide mappings from many characters to one character ? + For example sh → ш or дж → c ? +How might you make different mapping rules for characters at the beginning or end of the string ? + + +### Правила для транслитерации + +Основная идея - это начинать транслитерацию со сложных, многобуквенных преобразований (ч - tch). Например: +>Шарик -- sh-арик -- sharik + +Далее нужно заменить все гласные в начале и в конце слова (Я - ya). +>яблоко -- ya-блоко -- yabloko + +После чего уже можно переходить на простые однобуквенные преобразования (у - u) +>мед -- med + +## Методы +### Кодировка-декодировка с помощью KOI-8R + +Транслитерация с помощью кодировки KOI-8R - не самый эффективный метод транслитерации текста. +Но он обеспечивает некоторые особенности, которые не доступны другим способам: + + 1)Возможность восстановить первоначальный текст + + 2)Правила кодировки уже заданы + +Метод транслитерации с помощью KOI-8R представлен в файле transliterate_koi8r.py + +### Кодировка-декодировка с помощью правил + +Правила для транслетерации находятся в файле rules.txt. + В нем заданы правила для согланых, гласных, а также для гласных в начале слова. + + +
diff --git a/2018-komp-ling/practicals/xrenner-response.md b/2018-komp-ling/practicals/xrenner-response.md new file mode 100644 index 00000000..4534f8de --- /dev/null +++ b/2018-komp-ling/practicals/xrenner-response.md @@ -0,0 +1,34 @@ +# Xrenner Response + +So, the first - we must make all preparation, like install the xrenner etc: +The standart model for english language works just fine: + + $ python3 xrenner.py -m eng -o html example_in.conll10 > /mnt/c/sub_wsl/example.html + +Next, we must make our language model (in this case - russian). We can make this model from scratch, or copy a meta-language folder: + + $ cp -R ./models/udx ./models/rus + +### Rules + +Add some rules to our model. + +__pronouns.tab:__ + + я 1sg + мы 1pl + он male + она fema + его male + её fema + меня 1sg + нас 1pl + +__coref.tab:__ + + Рабиндранат Тагор|Тагор coref + +This simple rules give us a great results (see: pushkin.html): + + $ python3 xrenner.py -m rus -o html pushkin.conllu > /mnt/c/sub_wsl/pushkin.html + diff --git a/2018-komp-ling/practicals/xrenner_practical/example.html b/2018-komp-ling/practicals/xrenner_practical/example.html new file mode 100644 index 00000000..b3d25489 --- /dev/null +++ b/2018-komp-ling/practicals/xrenner_practical/example.html @@ -0,0 +1,540 @@ + + + + + + + + + + +
+New +Zealand +
+begins +process +to +consider +changing +national +flag +design +
+Thursday +
+, +
+May +7 +, +2015 +
+On +Tuesday +, +
+the +
+New +Zealand +
+government +
+announced +the +start +of +
+a +public +process +to +suggest +
+designs +for +
+a +new +national +flag +
+
+, +and +determine +whether +
+their +
+citizens +would +prefer +a +different +national +flag +over +
+the +current +one +
+
+. +
+The +current +flag +of +
+New +Zealand +
+
+. +
+The +current +
+New +Zealand +
+flag +
+is +partially +based +on +
+the +United +Kingdom +'s +flag +
+; +
+the +new +one +
+would +be +unique +to +
+New +Zealand +
+. +
+The +government +'s +
+Flag +
+Consideration +Project +
+has +planned +a +number +of +conferences +and +roadshows +as +part +of +
+this +process +
+, +with +the +first +meeting +set +to +take +place +in +Christchurch +on +May +16 +. +According +to +the +
+New +Zealand +
+Herald +, +
+
+Emeritus +Professor +John +Burrows +
+, +
+the +chairman +of +the +
+project +
+'s +panel +of +twelve +
+, +
+said +
+New +Zealand +
+'s +flag +has +never +before +been +open +to +public +choice +. +
+Professor +Burrows +
+also +said +resources +and +kits +would +be +accessible +for +schools +and +communities +, +" +For +example +, +
+schools +
+can +run +
+their +
+own +flag +discussions +and +referendums +to +mirror +the +formal +process +as +part +of +
+their +
+own +learning +exercise +" +. +
+People +
+were +encouraged +to +submit +
+their +
+designs +online +at +www.flag.govt.nz +and +suggest +what +
+the +flag +
+should +mean +on +www.standfor.co.nz +. +
+Names +of +participants +
+would +be +engraved +, +at +
+their +
+option +, +on +a +flag +pole +monument +to +be +built +in +
+the +nation +'s +capital +
+, +
+Wellington +
+. +
+New +Zealand +'s +Prime +Minister +John +Key +
+said +
+he +
+believes +redesigning +
+the +flag +
+now +has +a +" +strong +rationale +" +. +
+Mr +Key +
+promoted +the +campaign +for +a +unique +
+New +Zealand +
+flag +on +
+Waitangi +Day +
+- +
+February +6 +
+- +this +year +. +Of +the +public +process +, +
+he +
+said +, +" +In +the +end +
+I +
+'ll +have +one +vote +in +each +referendum +just +like +every +other +New +Zealander +on +the +electoral +roll +" +. +
+The +
+New +Zealand +
+government +
+intends +to +hold +two +referendums +to +reach +a +verdict +on +
+the +flag +
+, +at +an +estimated +cost +of +NZ +$ +26 +million +, +although +a +recent +poll +found +only +a +quarter +of +citizens +favoured +changing +
+the +flag +
+. +This +is +a +decrease +from +the +year +before +, +when +it +was +forty +percent +. +The +first +referendum +is +to +be +held +from +November +20 +to +December +11 +, +selecting +a +single +new +flag +design +out +of +about +four +finalists +. +
+Voters +
+would +then +choose +between +
+the +new +flag +
+and +
+
+their +
+current +flag +
+early +in +2016 +. + + + \ No newline at end of file diff --git a/2018-komp-ling/practicals/xrenner_practical/pushkin.html b/2018-komp-ling/practicals/xrenner_practical/pushkin.html new file mode 100644 index 00000000..c9fe846a --- /dev/null +++ b/2018-komp-ling/practicals/xrenner_practical/pushkin.html @@ -0,0 +1,133 @@ + + + + + + + + + + +Однажды +
+Пушкин +
+написал +
+письмо +
+
+
+Рабиндранату +
+Тагору +
+. +" +
+Дорогой +далекий +друг +
+, +— +писал +
+он +
+, +— +
+я +
+
+Вас +
+не +знаю +, +и +
+Вы +
+
+меня +
+не +знаете +. +Очень +хотелось +бы +познакомиться +. +Всего +хорошего +. +
+Саша +
+" +. +Когда +
+письмо +
+принесли +, +
+Тагор +
+предавался +
+самосозерцанию +
+. +Так +погрузился +, +
+хоть +режь +
+его +
+
+. +
+
+Его +
+жена +
+толкала +, +толкала +, +
+письмо +
+подсовывала +— +не +видит +. +
+Он +
+, +правда +, +по-русски +читать +не +умел +. +Так +и +не +познакомились +. + + + \ No newline at end of file diff --git a/2018-komp-ling/practicals/xrenner_practical/xrenner-response.md b/2018-komp-ling/practicals/xrenner_practical/xrenner-response.md new file mode 100644 index 00000000..4534f8de --- /dev/null +++ b/2018-komp-ling/practicals/xrenner_practical/xrenner-response.md @@ -0,0 +1,34 @@ +# Xrenner Response + +So, the first - we must make all preparation, like install the xrenner etc: +The standart model for english language works just fine: + + $ python3 xrenner.py -m eng -o html example_in.conll10 > /mnt/c/sub_wsl/example.html + +Next, we must make our language model (in this case - russian). We can make this model from scratch, or copy a meta-language folder: + + $ cp -R ./models/udx ./models/rus + +### Rules + +Add some rules to our model. + +__pronouns.tab:__ + + я 1sg + мы 1pl + он male + она fema + его male + её fema + меня 1sg + нас 1pl + +__coref.tab:__ + + Рабиндранат Тагор|Тагор coref + +This simple rules give us a great results (see: pushkin.html): + + $ python3 xrenner.py -m rus -o html pushkin.conllu > /mnt/c/sub_wsl/pushkin.html + diff --git a/2018-komp-ling/quizzes/quiz-1/quiz-1-response.md b/2018-komp-ling/quizzes/quiz-1/quiz-1-response.md new file mode 100644 index 00000000..714bc52c --- /dev/null +++ b/2018-komp-ling/quizzes/quiz-1/quiz-1-response.md @@ -0,0 +1,51 @@ + + +
+ +# Quiz 1 + +1. Which problems does maxmatch suffer from? (Choose all that + apply.) + + a) requires comprehensive dictionary + + d) constructs non-grammatical sentences + +2. Write a perl/sed substitution with regular expressions that + adds whitespace for segmentation around "/" in "either/or" + expressions but not around fractions "1/2": + Answer: + + sed 's/[[:alpha:]][/][[:alpha:]]/ \/ /' + +3. the text mentions several times that machine learning + techniques produce better segmentation than rule-based + systems; what are some downsides of machine learning + techniques compared to rule-based? + Answer: + 1) model overfitting + 2) Mono-language + 3) Impossibility of interpretation + +4. write a sentence (in English or in Russian) which maxmatch + segments incorrectly. + Answer: + + При правовых вопросах + + Приправовыхвопросах + + Приправ о вы х вопросах + + +5. what are problems for sentence segmentation? provide one + example in English or Russian for each that applies. + + a) ambiguous abbrevations with punctuation + + c) sentences lacking separating punctuation + +
diff --git a/2018-komp-ling/quizzes/quiz3/quiz-3-response.md b/2018-komp-ling/quizzes/quiz3/quiz-3-response.md new file mode 100644 index 00000000..916a0618 --- /dev/null +++ b/2018-komp-ling/quizzes/quiz3/quiz-3-response.md @@ -0,0 +1,65 @@ + + +
+ +# Quiz 3 + +1. In the reading, it is claimed that to implement a morphological disambiguator for an unseen language, it takes roughly the same amount of time whether annotating a corpus to train on versus writing constraint grammar rules. + + a) Give an argument for why constraint grammar rules are more valuable + + Constraint grammar rules gives us a great precision score, but a recall score is low most of the time. + So, if constraint rules can be simple implemented – we should use constraint grammar. + + b) Give an argument for why corpus annotation and HMM training is more valuable + + Counterwise, HMM gives us a great Recall score (bigger than CG rules), + but HMM will never reach the Precision level of CG rules + + +2. Can the two systems be used together? Explain. + + Yes. The basis of the grammar is composed of constraint rules. Yet, when rules + cannot provide a solution, there is room for the use of elements that contain + probabilistic features; this contributes to robustness in the grammar. + So, first of all we should use CG rules, and after that – HMM. + +3. Give a sentence with morphosyntactic ambiguity. +What would you expect a disambiguator to do in this situation? What can you do? + + +‘Косой косой косил косой’ + +In this case disambiguator will give us + + [A,A,V,N] + +because for a verb there must be a NOUN. +But the real PoS tags are + + [A,N,V,N] + +4. Choose several (>2) quantities that evaluate the quality of a morphological disambiguator, +and describe how to compute them. Describe what it would mean to have disambiguators which +differ in quality based on which quantity is used. + + Difficulty in evaluate the proper score for FP,FN e.t.c is what we need to summarize + all the answers for every tag we have. Example: take all NOUN tag from golden standard + and from the answers of our model. If : + +• Standard and answers equal = TP + +• Standard is NOUN, but answer is not = FN + +• Answer is NOUN, but standard is not = FP + +• Not Standard, nor answer is NOUN = TN + +Next, we summarize all answers and calculate Precision and Recall. + +5. Give an example where an n-gram HMM performs better than a unigram HMM tagger. + +