CS224N Assignment1 Section 1

运行环境需求

 # All Import Statements Defined Here
 # Note: Do not add to this list.
 # All the dependencies you need, can be installed by running .
 # ----------------

 import sys
 assert sys.version_info[0]==3
 assert sys.version_info[1] >= 5

 from gensim.models import KeyedVectors
 from gensim.test.utils import datapath
 import pprint
 import matplotlib.pyplot as plt
 plt.rcParams['figure.figsize'] = [10, 5]
 import nltk
 nltk.download('reuters')
 from nltk.corpus import reuters
 import numpy as np
 import random
 import scipy as sp
 from sklearn.decomposition import TruncatedSVD
 from sklearn.decomposition import PCA

 START_TOKEN = '<START>'
 END_TOKEN = '<END>'

 np.random.seed(0)
 random.seed(0)
 # ----------------

Question 1.1: Implement distinct_words [code] (2 points)

Write a method to work out the distinct words (word types) that occur in the corpus. You can do this with for loops, but it's more efficient to do it with Python list comprehensions. In particular, this may be useful to flatten a list of lists. If you're not familiar with Python list comprehensions in general, here's more information.

You may find it useful to use Python sets to remove duplicate words.

 def distinct_words(corpus):
     """ Determine a list of distinct words for the corpus.
         Params:
             corpus (list of list of strings): corpus of documents
         Return:
             corpus_words (list of strings): list of distinct words across the corpus, sorted (using python 'sorted' function)
             num_corpus_words (integer): number of distinct words across the corpus
     """
     corpus_words = []
     num_corpus_words = -1

     # ------------------
     # Write your implementation here.
     raw_corpus_words = [word for corpu in corpus for word in corpu]
     corpus_words = list(set(raw_corpus_words))
     corpus_words = sorted(corpus_words)
     num_corpus_words = len(corpus_words)

     # ------------------

     return corpus_words, num_corpus_words

Question 1.2: Implement compute_co_occurrence_matrix [code] (3 points)

Write a method that constructs a co-occurrence matrix for a certain window-size