With the given parameters and initial conditions, r and tet are always zero.
In[1]:= c = 100;
b = 1;
a = 0.99;
w = NDSolve[{r'[t] ==
r[t]*(Cos[tet[t]]^2 + a*Sin[tet[t]]^2) -
c*r[t]^2*Sin[tet[t]]*Cos[tet[t]]*(Cos[tet[t]] - a*Sin[tet[t]]) -
r[t]^2*(Cos[tet[t]]^3 + a*b*Sin[tet[t]]^3),
tet'[t] == (a - 1)*Sin[tet[t]]*Cos[tet[t]] -
c*r[t]*Sin[tet[t]]*Cos[tet[t]]*(a*Cos[tet[t]] - Sin[tet[t]]) -
r[t]*Sin[tet[t]]*Cos[tet[t]]*(a*b*Sin[tet[t]] - Cos[tet[t]]), r[0] == 0,
tet[0] == 0}, {r[t], tet[t]}, {t, 0, 1000}]
Out[4]= {{r[t] -> InterpolatingFunction[][t], tet[t] -> InterpolatingFunction[][t]}}
(* This samples only every 100th point. All 1001 were zero. *)
In[5]:= Table[ Evaluate[{r[t], tet[t]} /. w[[1]]], {t, 0, 1000, 100}]
Out[5]= {{0., 0.}, {0., 0.}, {0., 0.}, {0., 0.}, {0., 0.}, {0., 0.}, {0., 0.},
{0., 0.}, {0., 0.}, {0., 0.}, {0., 0.}}
(* r' and tet' are zero at the beginning, and it looks like they
never get the chance to grow. *)
In[6]:= Eliminate[{r'[t] ==
r[t]*(Cos[tet[t]]^2 + a*Sin[tet[t]]^2) -
c*r[t]^2*Sin[tet[t]]*Cos[tet[t]]*(Cos[tet[t]] - a*Sin[tet[t]]) -
r[t]^2*(Cos[tet[t]]^3 + a*b*Sin[tet[t]]^3),
tet'[t] == (a - 1)*Sin[tet[t]]*Cos[tet[t]] -
c*r[t]*Sin[tet[t]]*Cos[tet[t]]*(a*Cos[tet[t]] - Sin[tet[t]]) -
r[t]*Sin[tet[t]]*Cos[tet[t]]*(a*b*Sin[tet[t]] - Cos[tet[t]]), r[0] == 0,
tet[0] == 0} /. t -> 0, {r, tet}]
Out[6]= r[0] == 0. && tet[0] == 0. && r'[0] == 0. &&
100. tet'[0] == -1. Cos[tet[0]] Sin[tet[0]]
In[7]:= FullSimplify[%]
Out[7]= r[0] == 0 && tet[0] == 0 && r'[0] == 0 && tet'[0] == 0
