网络爬虫-基本使用(代码片段)

    // 1. 打开浏览器,创建HttpClient对象    
    CloseableHttpClient httpClient = HttpClients.createDefault();
    
    // 2.输入网址，发起get请求创建HttpGet对象
    HttpGet get = new HttpGet("http://112.124.1.187/index.html?typeId=16");
    
    // 3.发情请求，返回响应，使用HttpClient对象发起请求
    CloseableHttpResponse response = httpClient.execute(get);
    
    // 4.解析响应，获取数据
    if(response.getStatusLine().getStatusCode() == 200)
        HttpEntity entity = response.getEntity();
        String content = EntityUtils.toString(entity,"utf-8");
        System.out.println(content);
    

    // 1. 打开浏览器,创建HttpClient对象
    CloseableHttpClient httpClient = HttpClients.createDefault();
    try 
        // 地址：http://112.124.1.187/index.html?typeId=16.带有参数
        // 创建URIBuilder
        URIBuilder uriBuilder = new URIBuilder("http://112.124.1.187/index.html");
        // 添加参数
        // 多个参数可以连着添加,在后面连着setParameter(key,value)
        uriBuilder.setParameter("typeId","16");
        // 2.输入网址，发起get请求创建HttpGet对象
        HttpGet get = new HttpGet(uriBuilder.build());
        // 3.发情请求，返回响应，使用HttpClient对象发起请求
        CloseableHttpResponse response = null;
        try 
            response = httpClient.execute(get);
            // 4.解析响应，获取数据
            if(response.getStatusLine().getStatusCode() == 200)
                HttpEntity entity = response.getEntity();
                String content = EntityUtils.toString(entity,"utf-8");
                System.out.println(content);
            
         catch (IOException e) 
            e.printStackTrace();
        
     catch (URISyntaxException e) 
        e.printStackTrace();
    

    // 1. 打开浏览器,创建HttpClient对象
    CloseableHttpClient httpClient = HttpClients.createDefault();
    
    // 地址：http://112.124.1.187/index.html?typeId=16.带有参数
    // 2.输入网址，发起post请求创建HttpPost对象
    HttpPost post = new HttpPost("http://112.124.1.187/index.html");
    // 2.1 声明List集合，封装表单中的参数
    List<NameValuePair> params = new ArrayList<>();
    // 2.2 添加参数
    params.add(new BasicNameValuePair("typeId","16"));
    // 2.3 创建表单的Entity对象，对参数进行url编码
    UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params,"utf-8");
    // 2.4 设置表单的Entity对象到Post请求中
    post.setEntity(formEntity);
    
    // 3.发情请求，返回响应，使用HttpClient对象发起请求
    CloseableHttpResponse response = null;
    try 
        response = httpClient.execute(post);
        // 4.解析响应，获取数据
        if(response.getStatusLine().getStatusCode() == 200)
            HttpEntity entity = response.getEntity();
            String content = EntityUtils.toString(entity,"utf-8");
            System.out.println(content);
        
     catch (IOException e) 
        e.printStackTrace();
     finally
        if(response != null)
            response.close();
        
        httpClient.close();
    

    public static void     main(String[] args) 
        // 创建连接池管理器
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
        // 设置最大连接数
        cm.setMaxTotal(10);
        // 设置每个主机最大连接数
        cm.setDefaultMaxPerRoute(2);
    
        // 使用连接池管理器发起请求
        doGet(cm);
        doGet(cm);
    
    
    private static void doGet(PoolingHttpClientConnectionManager cm) 
        // 从连接池中获取HttpClient对象
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
    
        HttpGet httpGet = new HttpGet("http://112.124.1.187");
        CloseableHttpResponse response = null;
        try 
            response = httpClient.execute(httpGet);
            if(response.getStatusLine().getStatusCode() == 200)
                String content = EntityUtils.toString(response.getEntity(),"utf-8");
                System.out.println(content.length());
            
         catch (IOException e) 
            e.printStackTrace();
         finally
            if(response != null)
                try 
                    response.close();
                 catch (IOException e) 
                    e.printStackTrace();
                
            
            // 不用关闭HttpClient，交由池来管理
            // httpClient.close();
        
    

    HttpGet get =     new HttpGet("http://112.124.1.187/index.html?typeId=16");
    // 配置请求信息
    RequestConfig config = RequestConfig.custom().setConnectTimeout(10000)  // 创建连接的最长时间，单位是毫秒
                            .setConnectionRequestTimeout(500)   // 设置获取连接的最长时间，单位是毫秒
                            .setSocketTimeout(10 * 1000)    // 设置数据传输的最长时间，单位是毫秒
                            .build();
    // 将配置给请求
    get.setConfig(config);