JAVA

spring serve robots.txt

    @Override
    protected void configure(HttpSecurity http) throws Exception {
        http.authorizeRequests()
            .antMatchers("/robots.txt").permitAll()
    }

spring serve robots.txt

@RequestMapping(value = "/robots.txt")
public void robots(HttpServletRequest request, HttpServletResponse response) {
    try {
        response.getWriter().write("User-agent: *
Disallow: /
");
    } catch (IOException e) {
        CustomLogger.info(TAG, "robots", "robots(): "+e.getMessage());
    }
}