Java 爬虫

创建一个maven项目

导入依赖

<dependencies>  
    <dependency>  
        <groupId>org.jsoup</groupId>  
        <artifactId>jsoup</artifactId>  
        <version>1.10.2</version>  
    </dependency>  
    <dependency>  
        <groupId>com.alibaba</groupId>  
        <artifactId>fastjson</artifactId>  
        <version>1.2.51</version>  
    </dependency>  
</dependencies

创建一个类

public class Reptile {  
    public static void main(String[] args) throws IOException {  
      
    }  
}

我们要爬取的网页是36壁纸

我们请求一下这个网页

Connection connection = Jsoup  
// 设置URL  
.connect(WEB_URL)  
// 忽略解析不了的类型，强制解析，避免UnsupportedMimeTypeException  
.ignoreContentType(true)  
// 设置超时时间(ms)  
.timeout(60000);  
Document document = connection.get();  
List<String> srcs = document.getElementsByAttribute("lazysrc").stream().parallel().map(e -> e.attr("lazysrc")).collect(Collectors.toList());

可以看到数据已经获取到了

接下来我们把它遍历出来

srcs.forEach(imgUrl -> {  
    String imgUrlClear = imgUrl.substring(0, imgUrl.lastIndexOf(".238.390.jpg"));  
    downloadPicture(imgUrlClear, "E:\\壁纸\\jail\\" + imgUrlClear.substring(imgUrl.lastIndexOf("/") + 1));  
});

然后我们把它下载到本地

public static void downloadPicture(String imageUrl, String path) {  
    URL url = null;  
    try {  
        url = new URL(imageUrl);  
        DataInputStream dataInputStream = new DataInputStream(url.openStream());  
        FileOutputStream fileOutputStream = new FileOutputStream(new File(path));  
        ByteArrayOutputStream output = new ByteArrayOutputStream();  
        byte[] buffer = new byte[4096];  
        int length;  
        while ((length = dataInputStream.read(buffer)) > 0) {  
            output.write(buffer, 0, length);  
        }  
        fileOutputStream.write(output.toByteArray());  
        dataInputStream.close();  
        fileOutputStream.close();  
    } catch (IOException e) {  
        e.printStackTrace();  
    }  
}

这是完整代码

import org.jsoup.Connection;  
import org.jsoup.Jsoup;  
import org.jsoup.nodes.Document;  
  
import java.io.*;  
import java.net.URL;  
import java.util.List;  
import java.util.stream.Collectors;  
  
public class Reptile {  
    static String WEB_URL = "https://www.3gbizhi.com/tag/dongman/";  
  
    public static void main(String[] args) throws IOException {  
  
        Connection connection = Jsoup  
                // 设置URL  
                .connect(WEB_URL)  
                // 忽略解析不了的类型，强制解析，避免UnsupportedMimeTypeException  
                .ignoreContentType(true)  
                // 设置超时时间(ms)  
                .timeout(60000);  
        Document document = connection.get();  
        List<String> srcs = document.getElementsByAttribute("lazysrc").stream().parallel().map(e -> e.attr("lazysrc")).collect(Collectors.toList());  
        srcs.forEach(imgUrl -> {  
            String imgUrlClear = imgUrl.substring(0, imgUrl.lastIndexOf(".238.390.jpg"));  
            downloadPicture(imgUrlClear, "E:\\壁纸\\jail\\" + imgUrlClear.substring(imgUrl.lastIndexOf("/") + 1));  
        });  
    }  
  
    public static void downloadPicture(String imageUrl, String path) {  
        URL url = null;  
        try {  
            url = new URL(imageUrl);  
            DataInputStream dataInputStream = new DataInputStream(url.openStream());  
            FileOutputStream fileOutputStream = new FileOutputStream(new File(path));  
            ByteArrayOutputStream output = new ByteArrayOutputStream();  
            byte[] buffer = new byte[4096];  
            int length;  
            while ((length = dataInputStream.read(buffer)) > 0) {  
                output.write(buffer, 0, length);  
            }  
            fileOutputStream.write(output.toByteArray());  
            dataInputStream.close();  
            fileOutputStream.close();  
        } catch (IOException e) {  
            e.printStackTrace();  
        }  
    }  
  
}

这是下载的壁纸