facet_wrap text labelling issues with stat_fit_glance

I am wondering why the text is trending higher in the plots... it won't stay put with the facet_wrap or facet_grid. In a more complex dataset plot, the text is illegible because of the overlap.

Below is data and code to reproduce the plot and issue. Adding geom="text" to stat_fit_glance, results in Error: Discrete value supplied to continuous scale .

library(ggpmisc)
library(ggplot)

DF <- data.frame(Site = rep(LETTERS[20:24], each = 4),
                 Region = rep(LETTERS[14:18], each = 4),
                 time = rep(LETTERS[1:10], each = 10),
                 group = rep(LETTERS[1:4], each = 10),
                 value1 = runif(n = 1000, min = 10, max = 15),
                 value2 = runif(n = 1000, min = 100, max = 150))
DF$time <- as.numeric(DF$time)

plot1 <- ggplot(data=DF, 
                aes(x=time, y= value2,group=Site)) +
  geom_point(col="gray", alpha=0.5,
             name="") +
  geom_line(aes(group=Site),col="gray", alpha=0.5,
            name="") +
  geom_smooth(se=F, col="darkorange", alpha=0.8,
              name="", fill="orange",
              method="lm",formula=formula1) +
  theme_bw() + 
  theme(strip.text.x = element_text(size=10),
        strip.text.y = element_text(size=10, face="bold", angle=0),
        strip.background = element_rect(colour="black", fill="gray90"),
        axis.text.x = element_text(size=10),  # remove x-axis text
        axis.text.y = element_text(size=10), # remove y-axis text
        axis.ticks = element_blank(),  # remove axis ticks
        axis.title.x = element_text(size=18), # remove x-axis labels
        axis.title.y = element_text(size=25), # remove y-axis labels
        panel.background = element_blank(), 
        panel.grid.major = element_blank(),  #remove major-grid labels
        panel.grid.minor = element_blank(),  #remove minor-grid labels
        plot.background = element_blank()) + 
  labs(y="", x="Year", title = "")+ facet_wrap(~group)

plot1 + stat_fit_glance(method = "lm", label.x="right", label.y="bottom",
                        method.args = list(formula = formula1),
                        aes(label = sprintf('R^2~"="~%.3f~~italic(p)~"="~%.2f',
                                            stat(..r.squared..),stat(..p.value..))),
                        parse = TRUE)