The numbercommand key should include the labels.
\unprotect
\unexpanded\def\thecurrentfloatnumber
{\ifnofloatcaption \else \ifnofloatnumber \else
\ifx\currentfloatnumber\relax\else
\floatcaptionparameter\c!numbercommand
{\namedtaggedlabeltexts
\t!floatlabel \currentfloat
\t!floatnumber\currentfloat
{\clf_savedlistprefixednumber{\currentfloat}\currentfloatnumber\relax
\thecurrentfloatnumbersuffix}}%
\fi
\fi \fi}
\protect
\useMPlibrary[dum]
\setuplabeltext[en][figure={LEFT , RIGHT}]
\definereferenceformat[infigure][label=*]
\setupcaption
[figure]
[textcommand=\groupedcommand{<}{>},
numbercommand=\groupedcommand{[}{]}]
\starttext
\startplacefigure[reference=fig:dummy,title=Dummy figure]
\externalfigure[dummy]
\stopplacefigure
See \infigure[fig:dummy].
\stoptext