The shell-command works if I use the function as below.
(defun my-dired-open-execute ()
(interactive)
(let* ((file-name (dired-get-file-for-visit))
(file-ext (file-name-extension file-name))
(case-fold-search t))
(if (file-exists-p file-name)
(cond
((and (string-match "^r$" file-ext) (file-remote-p file-name))
(let ((default-directory (or (and (eq major-mode 'dired-mode)
(dired-current-directory))
default-directory)))
(shell-command
(format "nohup /bin/apps64/software/R/R CMD BATCH --no-restore --no-save --slave \"%s\" &"
(file-name-nondirectory file-name)))
))
(t (w32-shell-execute "open" file-name nil 1))))))