diff --git a/src/target/xtensa/xtensa.c b/src/target/xtensa/xtensa.c
index 1a402743f..f8c36b01d 100644
--- a/src/target/xtensa/xtensa.c
+++ b/src/target/xtensa/xtensa.c
@@ -2077,17 +2077,16 @@ int xtensa_read_memory(struct target *target, target_addr_t address, uint32_t si
 			/* Disable fast memory access instructions and retry before reporting an error */
 			LOG_TARGET_DEBUG(target, "Disabling LDDR32.P/SDDR32.P");
 			xtensa->probe_lsddr32p = 0;
-			res = xtensa_read_memory(target, address, size, count, albuff);
-			bswap = false;
+			res = xtensa_read_memory(target, address, size, count, buffer);
 		} else {
 			LOG_TARGET_WARNING(target, "Failed reading %d bytes at address "TARGET_ADDR_FMT,
 				count * size, address);
 		}
+	} else {
+		if (bswap)
+			buf_bswap32(albuff, albuff, addrend_al - addrstart_al);
+		memcpy(buffer, albuff + (address & 3), (size * count));
 	}
-
-	if (bswap)
-		buf_bswap32(albuff, albuff, addrend_al - addrstart_al);
-	memcpy(buffer, albuff + (address & 3), (size * count));
 	free(albuff);
 	return res;
 }