java下载谷歌街景图片数据
摘要:本文介绍了一个Java实现的Google街景图片下载工具StreetViewDownloader。该工具支持多线程下载,能够自动拼接512x512的图片块为全景图,并按位置ID分类存储。主要功能包括:1) 通过HTTP请求获取街景图片块;2) 动态扩展图片画布拼接全景;3) 支持代理设置和重试机制;4) 自动记录下载失败的位置ID。工具采用模块化设计,包含图片处理、网络请求、错误处理等组件,
·
package com.images.utils;
import lombok.Data;
import okhttp3.OkHttpClient;
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
/**
* 街景图片下载器类,用于从Google街景服务下载全景图片
* 支持多线程下载和图片拼接功能
*/
@Data
public class StreetViewDownloader {
// 日期格式化器,用于生成文件名中的时间戳
private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyMMdd_HHmmss");
// 图片块大小,Google街景图片通常以512x512的块进行传输
private static final int BLOCK_SIZE = 512;
// 用户代理字符串,用于模拟浏览器行为
private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
// 缩放级别,范围1-4
private final int zoom;
// 下载文件保存路径
private final Path downloadPath;
// 重试次数
private final int retry;
// 是否覆盖已存在的文件
private final boolean overwrite;
// 成功下载计数器
private final AtomicInteger successNum = new AtomicInteger(0);
// 错误位置ID字典,记录每个位置ID的失败次数
private final Map<String, Integer> errPanoIdDict = new ConcurrentHashMap<>();
// 未成功添加到队列的文件路径
private final Path noAddQueuePath;
// 失败的位置ID记录文件路径
private final Path failedPath;
// 代理服务器主机地址
private String proxyHost;
// 代理服务器端口
private int proxyPort;
// 代理类型(HTTP/SOCKS)
private String proxyType;
// HTTP客户端实例
private OkHttpClient httpClient;
/**
* 构造函数,初始化街景图片下载器
* @param zoom 缩放级别,范围1-4
* @param downloadPath 下载文件保存路径
* @param retry 重试次数
* @param overwrite 是否覆盖已存在的文件
*/
public StreetViewDownloader(int zoom, String downloadPath, int retry, boolean overwrite) {
// 验证缩放级别是否在允许范围内
if (zoom < 1 || zoom > 4) {
throw new IllegalArgumentException("Incorrect zoom size: " + zoom + ", only sizes 1-4 are allowed.");
}
this.zoom = zoom;
this.downloadPath = Paths.get(downloadPath);
this.retry = retry;
this.overwrite = overwrite;
// 获取父目录路径
Path parentDir = this.downloadPath.getParent();
// 生成时间戳
String timestamp = LocalDateTime.now().format(DATE_FORMAT);
// 构建未成功添加到队列的文件路径
this.noAddQueuePath = parentDir.resolve("need_add_queue2").resolve(timestamp + ".txt");
// 构建失败的位置ID记录文件路径
this.failedPath = parentDir.resolve("field_pano_id").resolve(timestamp + ".txt");
try {
// 创建必要的目录结构
Files.createDirectories(this.noAddQueuePath.getParent());
Files.createDirectories(this.failedPath.getParent());
Files.createDirectories(this.downloadPath);
} catch (IOException e) {
throw new RuntimeException("Failed to create directories", e);
}
}
/**
* 向右扩展图片
* @param img 原始图片
* @param pixels 扩展的像素数
* @return 扩展后的图片
*/
private BufferedImage increaseRight(BufferedImage img, int pixels) {
int width = img.getWidth();
int height = img.getHeight();
int newWidth = width + pixels;
// 创建新的图片对象,背景为浅灰色
BufferedImage result = new BufferedImage(newWidth, height, BufferedImage.TYPE_INT_RGB);
Graphics2D g = result.createGraphics();
g.setColor(new Color(250, 250, 250));
g.fillRect(0, 0, newWidth, height);
// 将原始图片粘贴到新图片的左侧
g.drawImage(img, 0, 0, null);
g.dispose();
return result;
}
/**
* 向下扩展图片
* @param img 原始图片
* @param pixels 扩展的像素数
* @return 扩展后的图片
*/
private BufferedImage increaseDown(BufferedImage img, int pixels) {
int width = img.getWidth();
int height = img.getHeight();
int newHeight = height + pixels;
// 创建新的图片对象,背景为浅灰色
BufferedImage result = new BufferedImage(width, newHeight, BufferedImage.TYPE_INT_RGB);
Graphics2D g = result.createGraphics();
g.setColor(new Color(250, 250, 250));
g.fillRect(0, 0, width, newHeight);
// 将原始图片粘贴到新图片的顶部
g.drawImage(img, 0, 0, null);
g.dispose();
return result;
}
/**
* 获取指定位置和坐标的图片块
* @param locationId 位置ID
* @param x X坐标
* @param y Y坐标
* @param zoom 缩放级别
* @param retryCount 重试次数
* @return 图片字节数据
* @throws IOException 当网络请求失败时抛出
* @throws InterruptedException 当线程被中断时抛出
*/
private byte[] getImgXY(String locationId, int x, int y, int zoom, int retryCount) throws IOException, InterruptedException {
// 检查重试次数是否已用完
if (retryCount < 0) {
throw new IOException("Max retries exceeded for location: " + locationId);
}
// 构建请求URL
String urlStr = String.format(
"https://streetviewpixels-pa.googleapis.com/v1/tile?cb_client=maps_sv.tactile&panoid=%s&x=%d&y=%d&zoom=%d&nbt=1&fover=2",
locationId, x, y, zoom
);
try {
// 配置代理服务器
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 7897));
URL url = new URL(urlStr);
System.out.println("requestUrl:" + urlStr);
// 创建HTTP连接
HttpURLConnection conn = (HttpURLConnection) url.openConnection(proxy);
conn.setRequestMethod("GET");
conn.setRequestProperty("user-agent", USER_AGENT);
// 获取响应码
int responseCode = conn.getResponseCode();
// 处理400错误(表示请求超出范围)
if (responseCode == 400) {
throw new IOException("HTTP 400 - Bad Request");
}
// 处理非200响应
if (responseCode != 200) {
throw new IOException("HTTP " + responseCode + " - Request failed");
}
// 读取响应数据
try (InputStream in = conn.getInputStream();
ByteArrayOutputStream out = new ByteArrayOutputStream()) {
byte[] buffer = new byte[1024];
int bytesRead;
while ((bytesRead = in.read(buffer)) != -1) {
out.write(buffer, 0, bytesRead);
}
return out.toByteArray();
}
} catch (IOException e) {
e.printStackTrace();
// 重试逻辑
if (retryCount > 0) {
int sleepTime = new Random().nextInt(3) + 1;
Thread.sleep(sleepTime * 1000);
return getImgXY(locationId, x, y, zoom, retryCount - 1);
}
throw e;
}
}
/**
* 下载街景全景图片
* @param locationId 位置ID
* @return 下载的图片文件路径
* @throws IOException 当网络请求或文件操作失败时抛出
* @throws InterruptedException 当线程被中断时抛出
*/
public String downloadStreetView(String locationId) throws IOException, InterruptedException {
BufferedImage panorama = null;
int x = 0, y = 0;
int endColumn = 0;
// 循环获取所有图片块并拼接
while (true) {
while (true) {
// 检查是否到达列末尾
if (y == endColumn && y != 0) {
break;
}
try {
// 获取当前坐标的图片块
byte[] imageData = getImgXY(locationId, x, y, zoom,retry);
BufferedImage image = ImageIO.read(new ByteArrayInputStream(imageData));
// 初始化全景图片
if (x == 0 && y == 0) {
panorama = new BufferedImage(BLOCK_SIZE, BLOCK_SIZE, BufferedImage.TYPE_INT_RGB);
Graphics2D g = panorama.createGraphics();
g.setColor(new Color(250, 250, 250));
g.fillRect(0, 0, BLOCK_SIZE, BLOCK_SIZE);
g.dispose();
} else if (endColumn == 0 && y != 0) {
// 向下扩展图片
panorama = increaseDown(panorama, BLOCK_SIZE);
}
// 将图片块粘贴到全景图中
Graphics2D g = panorama.createGraphics();
g.drawImage(image, x * BLOCK_SIZE, y * BLOCK_SIZE, null);
g.dispose();
y++;
// 短暂休眠,避免请求过于频繁
Thread.sleep(10);
} catch (IOException e) {
// 处理400错误(表示到达边界)
if (e.getMessage().contains("HTTP 400")) {
endColumn = y;
break;
}
e.printStackTrace();
throw e;
}
}
y = 0;
x++;
try {
// 获取下一列的图片块
byte[] imageData = getImgXY(locationId, x, y, zoom,retry);
BufferedImage image = ImageIO.read(new ByteArrayInputStream(imageData));
// 向右扩展图片
panorama = increaseRight(panorama, BLOCK_SIZE);
// 将图片块粘贴到全景图中
Graphics2D g = panorama.createGraphics();
g.drawImage(image, x * BLOCK_SIZE, y * BLOCK_SIZE, null);
g.dispose();
} catch (IOException e) {
// 处理400错误(表示到达边界)
if (e.getMessage().contains("HTTP 400")) {
break;
}
e.printStackTrace();
throw e;
}
}
// 创建保存目录(按位置ID的首字母分类)
String subDir = "f2" + locationId.charAt(0);
Path saveDir = downloadPath.resolve(subDir);
Files.createDirectories(saveDir);
// 保存图片文件
String fileName = locationId + ".jpg";
Path filePath = saveDir.resolve(fileName);
ImageIO.write(panorama, "JPEG", filePath.toFile());
// 设置文件权限(仅适用于类Unix系统)
try {
filePath.toFile().setReadable(true, false);
filePath.toFile().setWritable(true, false);
} catch (SecurityException e) {
System.err.println("Warning: Could not set file permissions: " + e.getMessage());
}
return filePath.toString();
}
/**
* 下载图片(带重试机制)
* @param locationId 位置ID
* @param retryCount 重试次数
* @return 下载的图片文件路径,失败时返回null
*/
public String download(String locationId, int retryCount) {
// 检查重试次数
if (retryCount <= 0) {
System.err.println("Download failed for: " + locationId);
return null;
}
try {
// 尝试下载图片
return downloadStreetView(locationId);
} catch (Exception e) {
// 下载失败,进行重试
System.err.println("Download failed for " + locationId + ", retrying... (" + (retryCount - 1) + " attempts left)");
try {
Thread.sleep(1000);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
return download(locationId, retryCount - 1);
}
}
/**
* 处理单个位置ID的下载任务
* @param locationId 位置ID
*/
public void processLocation(String locationId) {
System.out.println("Downloading image " + locationId + " num: " + successNum.get());
try {
// 下载图片
String imgPath = download(locationId, retry);
if (imgPath != null) {
// 尝试将图片路径添加到队列
boolean isOk = addToQueue(imgPath);
if (isOk) {
// 增加成功计数器
successNum.incrementAndGet();
System.out.println("写入已下载图片队列:" + imgPath);
} else {
System.err.println("写入已下载图片队列失败:" + imgPath);
// 记录未能添加到队列的文件路径
synchronized (this) {
try {
Files.write(noAddQueuePath, (imgPath + System.lineSeparator()).getBytes(),
java.nio.file.StandardOpenOption.CREATE,
java.nio.file.StandardOpenOption.APPEND);
} catch (IOException e) {
e.printStackTrace();
System.err.println("Failed to write to no-add-queue file: " + e.getMessage());
}
}
}
} else {
// 记录失败次数
int errorCount = errPanoIdDict.getOrDefault(locationId, 0) + 1;
errPanoIdDict.put(locationId, errorCount);
// 如果失败次数超过5次,记录到失败文件
if (errorCount > 5) {
synchronized (this) {
try {
Files.write(failedPath, (locationId + System.lineSeparator()).getBytes(),
java.nio.file.StandardOpenOption.CREATE,
java.nio.file.StandardOpenOption.APPEND);
} catch (IOException e) {
e.printStackTrace();
System.err.println("Failed to write to failed file: " + e.getMessage());
}
}
System.err.println("下载图片多次失败:" + locationId);
}
}
} catch (Exception e) {
e.getMessage();
System.err.println("下载" + locationId + "失败: " + e.getMessage());
}
}
/**
* 将消息添加到队列(简化实现)
* @param message 要添加的消息
* @return 是否添加成功
*/
private boolean addToQueue(String message) {
// 这里需要实现消息队列的插入逻辑
// 由于原Python代码使用了特定的消息队列工具,这里简化处理
System.out.println("Adding to queue: " + message);
return true;
}
/**
* 开始下载任务
* @param locationIds 位置ID列表
*/
public void start(List<String> locationIds) {
System.out.println("开始下载...");
// 使用多线程处理下载任务
List<Thread> threads = new ArrayList<>();
for (String locationId : locationIds) {
Thread thread = new Thread(() -> processLocation(locationId));
threads.add(thread);
thread.start();
// 控制并发数,避免请求过于频繁
try {
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
Thread.currentThread().interrupt();
break;
}
}
// 等待所有线程完成
for (Thread thread : threads) {
try {
thread.join();
} catch (InterruptedException e) {
e.printStackTrace();
Thread.currentThread().interrupt();
}
}
System.out.println("下载完成,成功下载: " + successNum.get() + " 张图片");
}
/**
* 主方法,程序入口点
* @param args 命令行参数
*/
public static void main(String[] args) {
// 默认参数
int zoom = 4;
String outputPath = "D:\\demo\\项目1\\测试\\数据\\demo\\照片数据\\streetView360";
// 解析命令行参数
for (int i = 0; i < args.length; i++) {
if ("--zoom".equals(args[i]) && i + 1 < args.length) {
zoom = Integer.parseInt(args[i + 1]);
i++;
} else if ("--out".equals(args[i]) && i + 1 < args.length) {
outputPath = args[i + 1];
i++;
}
}
// 检查存储目录
File outputDir = new File(outputPath);
if (!outputDir.exists()) {
outputDir.mkdirs();
}
// 创建下载器实例
StreetViewDownloader downloader = new StreetViewDownloader(4, outputPath, 3, false);
// 这里应该从消息队列获取位置ID,为了演示,我们使用示例ID
List<String> sampleLocationIds = Arrays.asList(
"izZ2EA6QrfMP4Pst-G526g",
"loSYTzmXfbrNUeOicEgWXA"
);
// 开始下载
downloader.start(sampleLocationIds);
}
}
需要修改代理的ip和端口号,在谷歌地图上选择街景图片的id进行下载,下载完成之后,会直接将小图合成一张大图
更多推荐
所有评论(0)