The product of the transpose of dict with dict is not of full rank and can't be inverted.
In[26]:= test = {{1, 2}, {3, 5}, {7, 11}};
In[27]:= ps1 = PseudoInverse[test]
Out[27]= {{-(19/7), -(10/7), 8/7}, {12/7, 13/14, -(9/14)}}
In[29]:= ps2 = Inverse[Transpose[test].test].Transpose[test]
Out[29]= {{-(19/7), -(10/7), 8/7}, {12/7, 13/14, -(9/14)}}
In[30]:= alpha = 4; word = 3;
dict = Partition[
Flatten[Tuples[IdentityMatrix[alpha], word]], (alpha*word)];
In[33]:= Dimensions[Transpose[dict].dict]
Out[33]= {12, 12}
In[35]:= MatrixRank[Transpose[dict].dict]
Out[35]= 10